[ Feb 15 00:05:21 Enabled. ] [ Feb 15 00:05:21 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Feb 15 00:05:21 Method "start" exited with status 0. ] [ Feb 15 00:05:21 Rereading configuration. ] [ Feb 15 00:05:21 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:21.616762341Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:247"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:21.67084061Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:254","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:21.67087436Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:21.783285301Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:21.792090167Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:21.799709539Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:264","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.405614578Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.543566128Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:117"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.543608219Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:129"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.543753422Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.543898545Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:162","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.543938136Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.543944816Z","hostname":"sock","pid":1671,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:268"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.544006837Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547112159Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:22.547121439Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547137129Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547144029Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.54718712Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547218551Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547226271Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547230381Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547234641Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547241741Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547247451Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547251802Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"None"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547257352Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547261712Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547266652Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547270282Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547273942Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:88"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.547309913Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity decreased","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.72675972Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:326","current":"245","previous":"256"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.726799501Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:90"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.726806011Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893459453Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893498044Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893509105Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893513415Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893520115Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893524205Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893528195Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893537785Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893541995Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893546665Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity increased","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893550335Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:336","current":"255","previous":"245"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893560895Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893612637Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 503808, free: 20937461760, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 491520, free: 20937474048, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } })} }"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893696478Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.89377462Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.89379498Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893810351Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893830151Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893849431Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893854332Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893900662Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.893910593Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.908685157Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:697"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.908710798Z","hostname":"sock","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.999928038s","next_cleanup":"Instant { tv_sec: 1689, tv_nsec: 731539276 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.90880383Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3345"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.90880963Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.908896221Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.908911822Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912062435Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912211177Z","hostname":"sock","pid":1671,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912457472Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:800"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912464532Z","hostname":"sock","pid":1671,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912532814Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912587605Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912610495Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912614826Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:860"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912638446Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912650766Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912661326Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912680197Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:901"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912683697Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912687857Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912721108Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.912733908Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.91334113Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.913476923Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.913482473Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.913522944Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.913532914Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.913667907Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.91382671Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.913894691Z","hostname":"sock","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, rack_id: fdf13508-1e61-48db-a883-3beaba642d42, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.914090825Z","hostname":"sock","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:55221","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, rack_id: fdf13508-1e61-48db-a883-3beaba642d42, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.914175857Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:364"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.914186327Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:379"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.914213637Z","hostname":"sock","pid":1671,"file":"sled-agent/src/server.rs:47"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.914225178Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.914229858Z","hostname":"sock","pid":1671,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Attempting to add queued disks","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.914233688Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:394","num_disks":"9"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.916812699Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3573"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.922911091Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.922985072Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.922991672Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.923009773Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.923121685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.923127355Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.955924819Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.963580251Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.99362494Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:342"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:22.993653361Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Looking for unencrypted datasets in oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.068901121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.091109634Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.091185515Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.091199075Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.091271347Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.09141515Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.09143016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.117099902Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.124456218Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.228437251Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.249969001Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.250010042Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.250016382Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.250041652Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.250191595Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.250197615Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.276542301Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.283985289Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.387935721Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.410543032Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.410583603Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.410592483Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.410620334Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.410751836Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.410759316Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.438332596Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.446248764Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.547599014Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.569521382Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.569581733Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.569588103Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.569628934Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.569778487Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.569784407Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.597606841Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.604943368Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.709208976Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.731635754Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.731696845Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.731742056Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.731779666Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.731911969Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.731918099Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.759618731Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.767320195Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.870611404Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.89250545Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.892581762Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.892591232Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.892610683Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.892751455Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.892757775Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.919824915Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:23.927470728Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.032101944Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.054808436Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.054857657Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.054885558Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.054893018Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.05502001Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.055026801Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.083798874Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.091559029Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.196524302Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.218972109Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.21901141Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.2190177Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.219063501Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.219189763Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.219204364Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.247963397Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.255525808Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.361830827Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.376923678Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.376964459Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.376969149Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:354"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.379503479Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }), Pool { name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, info: ZpoolInfo { name: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }), Pool { name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, info: ZpoolInfo { name: \"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 503808, free: 20937461760, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 491520, free: 20937474048, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }), Pool { name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, info: ZpoolInfo { name: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", size: 20937965568, allocated: 826368, free: 20937139200, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }), Pool { name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, info: ZpoolInfo { name: \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60\", size: 20937965568, allocated: 768000, free: 20937197568, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }), Pool { name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, info: ZpoolInfo { name: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 780288, free: 20937185280, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 768000, free: 20937197568, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } })} }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.399637661Z","hostname":"sock","pid":1671,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.401932217Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.419565578Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.422866664Z","hostname":"sock","pid":1671,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.422905235Z","hostname":"sock","pid":1671,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.427899084Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:369"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.427915335Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.427920975Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.490468592Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Mounting oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.709374826Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.72110432Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.721195031Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.721224532Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.721242782Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:24.888881845Z","hostname":"sock","pid":1671,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.426430971Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"Underlay Available","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.426941781Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:122","sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.426963221Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:178"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.426970632Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:186"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.441140984Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.441463341Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.441660774Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.442023932Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.442419569Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.442518251Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.442619884Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.442735196Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.442837478Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.44293344Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.443032702Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.443207565Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.443360518Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.443522902Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.443685375Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.443840568Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.443993921Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.444177395Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.444319587Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.444471721Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.444642784Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:25.444782847Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:28.318559239Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 29.644206ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:28.61306418Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 147.265773ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:28.81222209Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 174.367606ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:29.009327349Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 265.789477ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:29.366722555Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 947.14547ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:29.776472263Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 891.315424ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:30.957888546Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 764.141353ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:32.08319494Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.292792668s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:32.989842045Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.460618632Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1847"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.681983516Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:978","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.682049697Z","hostname":"sock","pid":1671,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.738506043Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.738537753Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.738545643Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:424"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.738585434Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.738625795Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:35.738846129Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"344.215786ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.738959122Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.739184766Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:137"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.739202226Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 192.168.1.30, infra_ip_last: 192.168.1.30, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100 }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.739208657Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3345"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.739218137Z","hostname":"sock","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:55221","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.741938091Z","hostname":"sock","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.741996272Z","hostname":"sock","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.742320929Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.74239716Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.742427281Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.742458891Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.742482062Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:735","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.778448549Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:40975"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.778752365Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.778850047Z","hostname":"sock","pid":1671,"uri":"/zpools","method":"GET","req_id":"0d25f025-b8a2-40dc-beaa-0a2c961e624b","remote_addr":"[fd00:1122:3344:101::1]:40975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":251,"response_code":"200"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.797660142Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.809865545Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34952"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.809965877Z","hostname":"sock","pid":1671,"uri":"/sled-role","method":"GET","req_id":"c472a43e-8e8f-4976-b3d8-ec672f841849","remote_addr":"[fd00:1122:3344:101::1]:34952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.812427166Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.812455027Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.812750673Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.812952867Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:748"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.841857173Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.842037476Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52275"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.842277821Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.842286321Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.894616945Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.894656756Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.919216675Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.946406837Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.946451548Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.994901784Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.994953135Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.994960595Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:35.994996886Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.040936752Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.040984033Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.040989583Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:36.087231655Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"556.361378ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Configuring new Omicron zone: oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.094225144Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.098167683Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.102177642Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.114303924Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.119031229Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.12310451Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.131441746Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.32427598Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.565722614Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:36.644974014Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"1.155028169s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:36.918422786Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:37.364458107Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:37.635420239Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:37.80098197Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"1.148608041s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:37.947800337Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:37.957867298Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:37.963395878Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249\") failed: Property not found. retry in 68.369584ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.080958141Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945\") failed: Property not found. retry in 35.203624ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.08840816Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796\") failed: Property not found. retry in 55.926181ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.094641144Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945\") failed: Property not found. retry in 126.088873ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.131296645Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249\") failed: Property not found. retry in 59.827301ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.223644886Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:38.223892201Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"wait for service Some(\"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796\") failed: Property not found. retry in 62.611857ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.251623384Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945\") failed: Property not found. retry in 161.438468ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.275471559Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249\") failed: Property not found. retry in 297.309658ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.306411346Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796\") failed: Property not found. retry in 261.156789ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.365175448Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945\") failed: Property not found. retry in 312.206425ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.462032859Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249\") failed: Property not found. retry in 344.780926ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.6191357Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796\") failed: Property not found. retry in 556.665116ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.645733171Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:38.661019106Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"wait for service Some(\"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945\") failed: Property not found. retry in 646.969673ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.789288503Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.951042368Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"4.667637681s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"wait for service Some(\"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249\") failed: Property not found. retry in 668.667693ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:38.9747458Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796\") failed: Property not found. retry in 739.773973ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:39.212411598Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:39.287605747Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"wait for service Some(\"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945\") failed: Property not found. retry in 914.675593ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:39.445862742Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249\") failed: Property not found. retry in 1.278064114s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:39.653836639Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:39.780460283Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1864"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:39.780485333Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796\") failed: Property not found. retry in 506.871528ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:39.963669636Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:40.07475426Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"wait for service Some(\"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945\") failed: Property not found. retry in 1.07307941s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:40.370431614Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796\") failed: Property not found. retry in 915.165315ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:40.480205893Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249\") failed: Property not found. retry in 867.801139ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:40.939653293Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:41.146348573Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"wait for service Some(\"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796\") failed: Property not found. retry in 1.054033863s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:41.404147413Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945\") failed: Property not found. retry in 1.070626145s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:41.450994156Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249\") failed: Property not found. retry in 976.840296ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:41.815666807Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:42.348564101Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"wait for service Some(\"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796\") failed: Property not found. retry in 1.169854314s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:42.468464601Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945\") failed: Property not found. retry in 1.382001619s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:42.530461097Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249\") failed: Property not found. retry in 1.076636895s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:42.801425689Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:43.079920621Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:43.619990528Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"7.40092571s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Ensuring address fd00:1122:3344:1::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:43.681241509Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:43.68129499Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:1::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.228883857Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.228945458Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe4d:ce1d","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.310614256Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1871"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.316287819Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.400545129Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2126"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.43723349Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2404"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.671887688Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2283"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.929374212Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.929653337Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.961623095Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2318"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.977619243Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:2::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:44.977661324Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:45.161170373Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2585"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:45.306558681Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2590"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:45.488854275Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2524"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:45.529744551Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:45.529775791Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:45.662972257Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2312"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:45.672767062Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3597","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:45.672819563Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3613"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:45.672828843Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:45.71380612Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2126"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.221832718Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3621"} {"msg":"configuring dendrite zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.228166584Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3677"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.269915317Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:3::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.269964298Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.422202533Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3717","rack_subnet":"fd00:1122:3344::"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.472535666Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:360"} {"msg":"Failed to get switch ID from MGS (retrying in 62.437847ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:46.504794009Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0a20033a-fe8e-45b3-b3a7-bff682b01c9e\", \"content-length\": \"152\", \"date\": \"Thu, 15 Feb 2024 00:05:46 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"0a20033a-fe8e-45b3-b3a7-bff682b01c9e\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 118.5065ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:46.568904297Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"36b656f1-c86d-418b-b713-7b9430fe9102\", \"content-length\": \"152\", \"date\": \"Thu, 15 Feb 2024 00:05:46 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"36b656f1-c86d-418b-b713-7b9430fe9102\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 184.549651ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:46.688978841Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a5921b3a-11f0-4df3-949f-4fb7e9927700\", \"content-length\": \"152\", \"date\": \"Thu, 15 Feb 2024 00:05:46 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"a5921b3a-11f0-4df3-949f-4fb7e9927700\" }"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.821506263Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.821551804Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.874984129Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:408"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.906867255Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:567"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.906949277Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:609"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.907466527Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:612","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.907535688Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:430","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n v4_routes: {},\n v6_routes: {},\n}"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.982267648Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:46.99642328Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2126"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.477078943Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.527361905Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.527412016Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.527434697Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.527497298Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.52760354Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.527842905Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.52810024Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"1abc9b2a-e982-4d0c-8c89-87896cc73b91","remote_addr":"[fd00:1122:3344:101::1]:52275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":11685984,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.529746963Z","hostname":"sock","pid":1671,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.578848612Z","hostname":"sock","pid":1671,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:47.623610264Z","hostname":"sock","pid":1671,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.790450276Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:453"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.790469996Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.822675799Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.82425844Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.82427071Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.852694747Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.852928742Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39411"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.853186437Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.853194427Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.873157995Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.873215596Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.890811157Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.890866488Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.908620012Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.908693604Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.908893337Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.909051521Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.920334245Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.953055898Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.954530337Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.954535537Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.999140066Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", _ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 214, 124]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:48.999186757Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:49.000630276Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:49.000638226Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:49.000883431Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:49.002271739Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:49.024014082Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:49.041914519Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:50.060581287Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7\") failed: Property not found. retry in 73.795842ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:50.190942076Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:50.260011773Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61982"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:50.260277069Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"3caffbb0-793e-4a72-9b08-dfbe578063d3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61982","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":63,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7\") failed: Property not found. retry in 137.526272ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:50.279813858Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7\") failed: Property not found. retry in 130.889811ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:50.431305468Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7\") failed: Property not found. retry in 373.882285ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:50.572618055Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7\") failed: Property not found. retry in 790.343089ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:50.955986778Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7\") failed: Property not found. retry in 1.316745307s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:51.757338684Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7\") failed: Property not found. retry in 1.026600876s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:53.082462591Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7\") failed: Property not found. retry in 684.52365ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:05:54.118102798Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:54.845792515Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::1b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:54.845853597Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:55.397132987Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:55.397190138Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Set up NTP service boundary=true, Servers=[\"0.pool.ntp.org\"]","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:55.591675455Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2209"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:05:55.618916748Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.257177283Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.257289955Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.25756725Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.257811965Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"02b951fc-5bbc-4c7f-91ca-516be61af134","remote_addr":"[fd00:1122:3344:101::1]:39411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":12404847,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.260443078Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:494"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.29013899Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.290315973Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53300"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.298905624Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3331"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.299044437Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"8afc686b-e55c-41ed-a08d-dbb5abb16738","remote_addr":"[fd00:1122:3344:101::1]:53300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8483,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:01.301601698Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.431032198Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.431199852Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:36721"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.439703451Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"6d2aee67-dd90-4945-9b4b-c3f04f1fb19e","remote_addr":"[fd00:1122:3344:101::1]:36721","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8441,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.442741602Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:01.442761002Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.557847827Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.55802798Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63723"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.566599801Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"48772b44-2a2c-4ff8-bdbf-dfaefef3fbca","remote_addr":"[fd00:1122:3344:101::1]:63723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8533,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.570369196Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:01.570376947Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.751248902Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.751411316Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43421"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.759461526Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"5a2e1b08-48f4-4e56-ac97-d9e68f76f75d","remote_addr":"[fd00:1122:3344:101::1]:43421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8008,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.762408985Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:01.762418785Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.977777158Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.977964382Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53916"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.986216567Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"8e3a23a3-11d7-41a2-a065-df027a9ee0e9","remote_addr":"[fd00:1122:3344:101::1]:53916","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8216,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:01.989792638Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:01.989801968Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.27280038Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.272953113Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45950"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.282103216Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"bf99c6aa-d532-4fdf-b814-45ce202895fb","remote_addr":"[fd00:1122:3344:101::1]:45950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":9123,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.284361221Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:02.284367591Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.538705201Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.538866524Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.547598409Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"8ff96eac-f24d-4437-abc1-31f63bdb93e1","remote_addr":"[fd00:1122:3344:101::1]:43200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8674,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.55117162Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:02.55117729Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.929955711Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.930115304Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:41938"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.938120044Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"25dca2e1-3367-4845-8917-2bb7e59e571c","remote_addr":"[fd00:1122:3344:101::1]:41938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7979,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:02.941603064Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:02.941614224Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:03.447603461Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:03.447770994Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38247"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:03.457252213Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"54289073-9430-4acf-a12e-aa5d178c8685","remote_addr":"[fd00:1122:3344:101::1]:38247","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":9443,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:03.459328995Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:03.459334495Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:03.856970982Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:03.857113955Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58137"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:03.866019993Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"6dde0ebf-e651-493c-872d-7dc6ab23bb57","remote_addr":"[fd00:1122:3344:101::1]:58137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8819,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:03.869258077Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:03.869267218Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:04.482030113Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:04.482138135Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47240"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:04.489762917Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"1f2b9fa0-8d0b-4116-babb-f80e95e99b6c","remote_addr":"[fd00:1122:3344:101::1]:47240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7494,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:04.491680446Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:04.491685836Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:05.222551326Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:05.2227541Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:56625"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:05.231372262Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"30914209-b0a8-4168-8b48-e8d308578f64","remote_addr":"[fd00:1122:3344:101::1]:56625","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8564,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:05.233365322Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:05.233373322Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:05.934218534Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:05.934391998Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:59941"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:05.942833016Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"106bba53-9f53-499b-9af6-2b898346a6f2","remote_addr":"[fd00:1122:3344:101::1]:59941","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8393,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:05.945881087Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:05.945888817Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:07.479670703Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:07.479855046Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:55126"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:07.48305585Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:07.487925687Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"1ac36ca6-4ed0-4349-a8a2-7826fac8d2b7","remote_addr":"[fd00:1122:3344:101::1]:55126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8039,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:07.490790594Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:07.490803094Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.409973574Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.410139887Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48944"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.413392022Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.432172886Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"8379a709-1ebd-4ec6-8be7-49c61235ac22","remote_addr":"[fd00:1122:3344:101::1]:48944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":21978,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380539, ip_addr: 162.159.200.123, stratum: 4, ref_time: 1707955567.670102, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.435728357Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.435735827Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.464627993Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.464900369Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61545"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.465236166Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.465249006Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.485089061Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.485133022Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.503549149Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.50359097Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.521726092Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.521766793Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.572580566Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.572623056Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.624604403Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.624662034Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.676848054Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.676891015Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.728450043Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.728494054Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.77945691Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.779497981Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.779693695Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.779826057Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:08.812303985Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:08.812353426Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:08.812365256Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:08.812374406Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:08.812383626Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.812497518Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"edb9ffcc-9b38-45cd-94cc-028ce84b55d0","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized)]","latency_us":347435,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:08.812689292Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"138.025882ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"edb9ffcc-9b38-45cd-94cc-028ce84b55d0\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"edb9ffcc-9b38-45cd-94cc-028ce84b55d0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.951813296Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.952296105Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.952307816Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.97055046Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.970614551Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.988567189Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:08.98860862Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.006792617Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.006834093Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.02525188Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.025290009Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.042887964Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.042928596Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.060080177Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.060185084Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.077744495Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.077782465Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.095501685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.095541914Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.095722049Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.09584402Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.130199363Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.130257151Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.130266657Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.130274804Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.130346897Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.130449856Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"77ff47e8-36b8-411f-89f9-d8eb1fe33e16","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized)]","latency_us":177919,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.13063661Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"102.961201ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77ff47e8-36b8-411f-89f9-d8eb1fe33e16\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77ff47e8-36b8-411f-89f9-d8eb1fe33e16\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.235063528Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.235718605Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.235745746Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.255062404Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.255112439Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.272826942Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.272876649Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.290237815Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.290290681Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.30854757Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.308600492Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.32654851Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.326602427Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.34484334Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.344960385Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.362285986Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.36233454Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.38046777Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.380522212Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.38077281Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.380910344Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.413037307Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.413136094Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.413142141Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.413147185Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.413152031Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.413249768Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"428475d9-f4b2-4afa-b141-7e173f2f14ee","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized)]","latency_us":177256,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.413417865Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"162.528069ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"428475d9-f4b2-4afa-b141-7e173f2f14ee\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"428475d9-f4b2-4afa-b141-7e173f2f14ee\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.577205055Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.577862307Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.577886252Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.596835104Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.596877086Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.614757812Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.614794853Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.632455667Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.632492531Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.650359211Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.650405797Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.668665372Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.668701157Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.686497128Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.686533531Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.704336487Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.704371907Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.722753559Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.722797538Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.723070876Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.723195079Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.755797179Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.755847552Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.755856083Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.755863995Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.755870192Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.756057192Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"7c063265-9072-4258-8cfb-38a840a42ed2","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized)]","latency_us":177893,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:09.756263268Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"184.045982ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7c063265-9072-4258-8cfb-38a840a42ed2\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7c063265-9072-4258-8cfb-38a840a42ed2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.942095374Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.943360386Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.943379783Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.962398781Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.962467979Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.980713417Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.980766407Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.999482624Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:09.999534197Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.021811754Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.021868089Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.040842215Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.040887853Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.05907591Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.059123826Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.078243374Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.07828409Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.096103173Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.096159537Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.096372101Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.09650894Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.12906377Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.129094643Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.129098721Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.129104599Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.129110627Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.129230133Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"995eef98-ab01-48bd-9219-66f4ef19f1f1","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized)]","latency_us":186089,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.129405139Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"106.190662ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"995eef98-ab01-48bd-9219-66f4ef19f1f1\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"995eef98-ab01-48bd-9219-66f4ef19f1f1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.236827471Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.23811468Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.238132662Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.258127261Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.2581699Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.275980944Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.276019765Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.293725787Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.293773047Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.311988305Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.312025243Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.330258802Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.33029455Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.348233922Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.348268404Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.366036121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.366076004Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.38421009Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.384250122Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.384455832Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.384587291Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.416753268Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.416865792Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.416872645Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.416880061Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.416887899Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.417037902Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"3bbf0581-5ce9-4f78-84ea-74666d41af36","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized)]","latency_us":179277,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.41724274Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"252.492621ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3bbf0581-5ce9-4f78-84ea-74666d41af36\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3bbf0581-5ce9-4f78-84ea-74666d41af36\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.671283929Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.672351282Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.672366685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.691401194Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.691459424Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.709128934Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.709182701Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.726691571Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.726752612Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.744345409Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.744399503Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.763383241Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.763435442Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.781291934Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.78134226Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.799145838Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.799195948Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.817801887Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.817930858Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.818193457Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.818318512Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.850760781Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.850792919Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.850799116Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.8508052Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.850810244Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:10.850907437Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"9b3b328b-7d9e-4569-8eca-6678ed9f5848","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized)]","latency_us":178706,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:10.851156593Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"270.849412ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b3b328b-7d9e-4569-8eca-6678ed9f5848\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9b3b328b-7d9e-4569-8eca-6678ed9f5848\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.123043868Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.123614442Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.123631458Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.142875267Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.142923249Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.161330773Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.161364065Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.17925424Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.179297309Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.197931107Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.197962739Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.216524857Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.216555336Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.234944383Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.235041247Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.252500153Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.252533163Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.270052718Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.270161287Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.270373195Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.270485688Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.302954064Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.30299134Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.302998634Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.303007568Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.303016709Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.30321332Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"06fa24a4-1a7e-4c90-adbd-a4ef6b232caf","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized)]","latency_us":179222,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.303430628Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"511.117192ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"06fa24a4-1a7e-4c90-adbd-a4ef6b232caf\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"06fa24a4-1a7e-4c90-adbd-a4ef6b232caf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.817325642Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.818114284Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.818129575Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.838072212Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.838123054Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.856944608Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.856995759Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.875171503Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.875218107Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.893446858Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.893496322Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.91225534Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.91230304Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.930024957Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.930072976Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.948351537Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.94840071Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.966611752Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.966665153Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.966901248Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:11.967126087Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.999916516Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.999960983Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.999967499Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.999975496Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:11.99998159Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.000172276Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"e8c131d1-895f-41ad-b910-57819b424fdd","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized)]","latency_us":181781,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:12.000352927Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"477.823174ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e8c131d1-895f-41ad-b910-57819b424fdd\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e8c131d1-895f-41ad-b910-57819b424fdd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.481083092Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.481724586Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.481740337Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.501116993Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.501163634Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.51917533Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.519231215Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.53622909Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.536272525Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.554277935Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.554320752Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.572137563Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.572180942Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.589488298Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.5895399Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.608338459Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.60837926Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.626001772Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.626043989Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.62622059Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.626335248Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:12.657793408Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:12.657900965Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:12.657909291Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:12.657914401Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:12.657935335Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:12.658045841Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"597649ff-c3d1-4fb3-bda2-3e6f06341379","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized)]","latency_us":176018,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:12.658230645Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"769.971992ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"597649ff-c3d1-4fb3-bda2-3e6f06341379\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"597649ff-c3d1-4fb3-bda2-3e6f06341379\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.431749403Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.432384634Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.432401725Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.452141278Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.452192617Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.470177966Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.470237706Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.488481898Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.488524405Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.5075277Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.507569748Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.526002261Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.526042274Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.544589056Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.544629679Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.563562188Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.563603889Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.581915746Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.58196656Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.582142916Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.58226049Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:13.615473004Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:13.615503164Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:13.615523292Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:13.615528655Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:13.615534617Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:13.615627394Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"437912f8-811f-42e6-a4ce-79f8c5aa9dae","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized)]","latency_us":182876,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:13.615799851Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"594.640615ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"437912f8-811f-42e6-a4ce-79f8c5aa9dae\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"437912f8-811f-42e6-a4ce-79f8c5aa9dae\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.212788089Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.213243946Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.213257681Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.233476567Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.233515483Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.251787816Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.251842051Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.270556048Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.270598565Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.28940635Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.289441826Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.307501248Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.307536574Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.325538731Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.325573896Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.34418188Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.344215191Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.362287274Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.362329303Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.362596899Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.362727119Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:14.395254185Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:14.395319145Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:14.39532807Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:14.395340904Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:14.395348883Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:14.395452563Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"ca536acb-ae50-4373-b072-67559f045dad","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized)]","latency_us":181878,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:14.395743325Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"891.620816ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ca536acb-ae50-4373-b072-67559f045dad\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ca536acb-ae50-4373-b072-67559f045dad\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.290649697Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.291377587Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.291401607Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.310226903Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.310264412Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.329716839Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.329753027Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.348809621Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.348854349Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.367209935Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.367242552Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.385432799Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.385473572Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.403437465Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.403471216Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.421242978Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.421342587Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.438983631Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.439024057Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.439228904Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.439417165Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:15.471766301Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:15.471814902Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:15.471823527Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:15.47183639Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:15.471844902Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:15.471954293Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"20edb31e-9670-4246-8f28-bc6a1d8793ec","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized)]","latency_us":180340,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:15.472135937Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"774.770631ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"20edb31e-9670-4246-8f28-bc6a1d8793ec\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"20edb31e-9670-4246-8f28-bc6a1d8793ec\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.250748875Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.251222004Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.251238954Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.271062029Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.273359719Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.288585497Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.28869488Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.306771938Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.306822057Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.32500596Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.325053305Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.342346938Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.342392877Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.360390882Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.360432142Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.377735925Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.377781526Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.395700038Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.395756879Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.396038401Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.396172804Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:16.427907418Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:16.427960462Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:16.427974741Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:16.427988194Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:16.428065807Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:16.428166835Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"bba73c52-4d75-4313-bef9-7e035c714d16","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized)]","latency_us":176549,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:16.428387386Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"908.329474ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bba73c52-4d75-4313-bef9-7e035c714d16\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bba73c52-4d75-4313-bef9-7e035c714d16\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.339797839Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.340401086Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.340413836Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.361089237Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.361136825Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.381296558Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.381333224Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.400337405Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.400376725Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.418441822Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.418478798Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.436904993Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.436941836Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.455443658Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.45548654Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.47427992Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.474317317Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.492188481Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.49223066Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.492429555Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.492558314Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:17.525158928Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:17.525207857Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:17.52521833Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:17.525227152Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:17.525238824Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:17.525358958Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"de2d49b1-619c-4d4a-ab92-25c19e876980","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized)]","latency_us":184605,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:17.52560184Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.467555042s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"de2d49b1-619c-4d4a-ab92-25c19e876980\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"de2d49b1-619c-4d4a-ab92-25c19e876980\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:18.999191414Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:18.999860766Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:18.999873656Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.019751682Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.019802327Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.039040406Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.039155307Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.05725822Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.057308115Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.074211757Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.074261998Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.092161707Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.092277585Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.110465635Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.110513364Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.128443061Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.128491896Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.147404474Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.147459469Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.14766021Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.147791482Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:19.17985211Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:19.179898602Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:19.179913321Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:19.179928265Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:19.179937152Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:19.180049824Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"acffbe91-ba51-4485-b039-77cdfbd929db","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized)]","latency_us":179897,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:19.18025439Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.052290343s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"acffbe91-ba51-4485-b039-77cdfbd929db\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"acffbe91-ba51-4485-b039-77cdfbd929db\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.236642177Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.237770417Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.237801786Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.257268675Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.257406615Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.275593193Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.275648216Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.293681382Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.293734268Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.311811804Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.311870727Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.328895904Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.328946202Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.347390224Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.347444488Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.365969943Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.366024703Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.383656975Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.383786073Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.384032002Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.38418734Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:20.416840735Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:20.416879754Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:20.416886739Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:20.416892879Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:20.416899142Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.416992031Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"f2023db3-d408-45ad-ad1e-ebcb370efcb6","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized)]","latency_us":179335,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:20.417162566Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.530985016s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f2023db3-d408-45ad-ad1e-ebcb370efcb6\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f2023db3-d408-45ad-ad1e-ebcb370efcb6\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.619893628Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37377"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:20.620575878Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"0155f83e-3d81-414c-aed7-f9584b1a1eb2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37377","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":611,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:21.953156078Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:21.953716321Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:21.953730628Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:21.97471387Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:21.974753396Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:21.992782141Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:21.992818404Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.010728239Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.010765121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.028915799Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.02896529Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.047047112Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.047085044Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.066572665Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.066610578Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.085182223Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.085218608Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.101888515Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.101933488Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.102135204Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.102266438Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:22.134205842Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:22.134262374Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:22.134278941Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:22.134290866Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:22.134300241Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:22.134476902Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"3540f5de-0515-4c97-abd2-839d41c992c3","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized)]","latency_us":180426,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:22.134698251Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.018835481s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3540f5de-0515-4c97-abd2-839d41c992c3\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3540f5de-0515-4c97-abd2-839d41c992c3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.160674359Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.161390687Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.161417341Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.183885865Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.184005032Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.202755561Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.202799015Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.221695469Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.22174498Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.24087406Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.240915705Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.259705597Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.259752567Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.277421685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.277476933Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.295768547Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.295825717Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.313884569Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.313961127Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.31416963Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.314369109Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:24.347781553Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:24.347835404Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:24.347842669Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:24.347852157Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:24.347866727Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:24.348010157Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"43f83293-b20f-45d5-8a08-18bee3f03214","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized)]","latency_us":186337,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:24.348216036Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.674943892s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"43f83293-b20f-45d5-8a08-18bee3f03214\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"43f83293-b20f-45d5-8a08-18bee3f03214\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.028372489Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.028908882Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.028987846Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.049713287Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.049773157Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.067633612Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.067690472Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.085722597Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.085772577Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.104450478Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.104509579Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.122252613Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.122300464Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.139992493Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.140050028Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.158629527Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.158673018Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.175566532Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.175617608Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.175812359Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.175939477Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:26.208184894Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:26.208232577Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:26.208238146Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:26.20825909Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:26.208267115Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:26.208349973Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"f4d4aca0-04b8-4399-9400-e5f11adcf655","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized)]","latency_us":179079,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:26.208605644Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"4.441972646s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f4d4aca0-04b8-4399-9400-e5f11adcf655\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f4d4aca0-04b8-4399-9400-e5f11adcf655\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.664709282Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.66531596Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.66532795Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.685739833Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.685800163Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.705028821Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.705071694Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.722751799Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.722789853Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.741760724Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.741796791Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.759417623Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.759454683Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.777656203Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.77769737Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.794978798Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.795013665Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.813455113Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.81350407Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.813731578Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.813925673Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:30.846197972Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:30.846244371Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:30.846258781Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:30.846273959Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:30.846289043Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:30.846383779Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"836c49c9-dd38-4739-806b-b1da78a7368c","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized)]","latency_us":180758,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:30.846595162Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.913907403s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"836c49c9-dd38-4739-806b-b1da78a7368c\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"836c49c9-dd38-4739-806b-b1da78a7368c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.769234769Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.769945542Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.769953783Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.791033699Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.791094197Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.809765691Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.809823752Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.828156547Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.82820737Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.846464581Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.846519436Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.865476146Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.865541623Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.883447172Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.883498558Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.901081522Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.901137408Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.91914447Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.919223353Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.919482829Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.91960736Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:33.951827645Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:33.951882594Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:33.95189091Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:33.951898841Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:33.951905207Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:33.952011915Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"842c74b0-7ffb-46ab-883c-31720e7fb4a3","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized)]","latency_us":181796,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:33.952268111Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"4.508646585s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"842c74b0-7ffb-46ab-883c-31720e7fb4a3\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"842c74b0-7ffb-46ab-883c-31720e7fb4a3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.474420595Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.475052586Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.475065599Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.496256407Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.496308468Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.514443205Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.514484653Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.532619919Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.532660635Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.552171942Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.552209648Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.571025482Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.571075779Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.588699741Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.588743907Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.606167216Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.606205982Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.624217623Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.624262024Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.624460478Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.624589575Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:38.656362584Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:38.656418009Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:38.656430498Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:38.656440257Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:38.656449539Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:38.656544415Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"57ba6f99-c3f6-4617-ba9e-70f6c8ef56dd","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized)]","latency_us":181210,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:38.656763898Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"7.776098451s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"57ba6f99-c3f6-4617-ba9e-70f6c8ef56dd\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"57ba6f99-c3f6-4617-ba9e-70f6c8ef56dd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.445124702Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.445794275Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.445809965Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.46658467Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.466661291Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.484081988Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.484136759Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.502788501Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.502857482Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.520127267Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.520186428Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.538938812Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.539004743Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.557367069Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.55741515Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.575718865Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.575770646Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.593962659Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.59402648Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.594253555Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.594381977Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:46.626824104Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:46.626884765Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:46.626893175Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:46.626905616Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:46.626912336Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:46.626991237Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"e4791c08-4804-40b0-b7ee-b20ab58eb73b","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized)]","latency_us":181425,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:46.627175831Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.995345707s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e4791c08-4804-40b0-b7ee-b20ab58eb73b\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e4791c08-4804-40b0-b7ee-b20ab58eb73b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.623217497Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.624557113Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.624569824Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.645611193Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.645659744Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.663557911Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.663596871Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.681473838Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.681510148Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.700084969Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.70013641Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.713048917Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.713333883Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"47ee68e7-70af-4a66-b2bd-8dbef797b4b5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60177","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":151,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.720673179Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.720682649Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.740345001Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.740394113Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.758474063Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.758523274Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.776119915Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.776172266Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.77639154Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.776510942Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:50.808895478Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:50.808954979Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:50.80896764Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:50.80897617Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:50.80898798Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:06:50.809103532Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"5dde27f6-c8b4-4682-b236-237c6b5a3548","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized)]","latency_us":185449,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:06:50.809361677Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"10.193353963s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5dde27f6-c8b4-4682-b236-237c6b5a3548\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:06:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5dde27f6-c8b4-4682-b236-237c6b5a3548\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.003724563Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.004449807Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.004459837Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.026506586Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.026570178Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.046371463Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.046426854Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.064448953Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.064515574Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.083133835Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.083188377Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.101466831Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.101525342Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.119673014Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.119721135Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.138311726Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.138375357Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.156557069Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.156637631Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.156887086Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.157024739Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:01.18916992Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:01.18921249Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:01.18922441Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:01.189230241Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:01.189236951Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:01.189343813Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"75e30880-83e5-4385-9e4e-58d44528bdaf","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized)]","latency_us":185106,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:01.189547937Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"10.043966205s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"75e30880-83e5-4385-9e4e-58d44528bdaf\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:07:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"75e30880-83e5-4385-9e4e-58d44528bdaf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.234487043Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.235637176Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.235642936Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.256375269Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.25641799Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.275562532Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.275604893Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.293394387Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.293440499Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.311730063Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.311769134Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.329492497Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.329529698Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.347529427Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.347571698Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.366199189Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.366237709Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.384469783Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.384517124Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.384778489Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.384915042Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:11.417632814Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:11.417696445Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:11.417704175Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:11.417713205Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:11.417730196Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:11.417873769Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8930c7c4-ca7d-4a86-b623-7582efd1db5c","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized)]","latency_us":182863,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:11.418105773Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"15.43107045s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8930c7c4-ca7d-4a86-b623-7582efd1db5c\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:07:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8930c7c4-ca7d-4a86-b623-7582efd1db5c\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:20.745326871Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38399"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:20.745440273Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"2f4cd75f-dfdf-4aee-9e6c-f3e67ec2d302","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38399","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":65,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.850042324Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.850708198Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.850717208Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.872221977Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.872286248Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.890864438Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.89094287Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.909611542Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.909675804Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.928156652Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.928210123Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.945847754Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.945907846Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.963925235Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.963979816Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.981819372Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:26.981879383Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:27.000174678Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:27.000260609Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:27.000555205Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:27.000706188Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:27.0328689Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:27.032928271Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:27.032959631Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:27.032965901Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:27.032972231Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:27.033068684Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c2dc2c0c-a09a-49a6-b9d2-ecf7d3afb7e7","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\", TimeNotSynchronized), (\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\", TimeNotSynchronized), (\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\", TimeNotSynchronized), (\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\", TimeNotSynchronized), (\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\", TimeNotSynchronized)]","latency_us":182602,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:27.033813338Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"16.492442452s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c2dc2c0c-a09a-49a6-b9d2-ecf7d3afb7e7\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:07:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c2dc2c0c-a09a-49a6-b9d2-ecf7d3afb7e7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.526818621Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.527530285Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.527539005Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.548710637Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.548774618Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.568023273Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.568089484Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.587499041Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.587554652Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.605270695Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.605330736Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.623282464Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.623347596Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.641943836Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.641994708Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.659482116Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.659542287Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.67770022Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.677769611Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.678079507Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.67824568Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.761589382Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.761661194Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.761668074Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.761674684Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.761679514Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.854412253Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.858688399Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.862534775Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.866972344Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.87178199Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.875955113Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.880877311Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.886113885Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.89035686Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:43.894605675Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:47.270862911Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:47.271037495Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:47.358132652Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:47.358294275Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 44.616981ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:47.409560147Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 75.854866ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:47.463563504Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 33.015312ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:47.577579448Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 119.995282ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:47.592305971Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:47.600015555Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:47.600617837Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 109.627679ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:47.635420921Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:47.647198046Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:47.649357439Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 145.944336ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:47.80436905Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 248.017387ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:47.840128023Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:47.851189674Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:47.861205993Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 334.608036ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.014321207Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 51.090608ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.10073445Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 440.576443ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.123914702Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 126.801922ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.175014951Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 125.456483ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.373810415Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 61.356786ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.43974435Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 838.481772ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.494534593Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 64.617116ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.573093149Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 1.333236879s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.639500183Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 65.750781ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.667583973Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 464.731426ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.709806875Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 58.046938ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.771346353Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 136.356603ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.817235108Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 259.365779ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.862204164Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 364.756567ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:48.988636246Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 235.107295ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:49.139550284Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 905.009034ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:49.190338727Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 1.374427352s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:49.349235225Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 887.361317ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:49.376254534Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 737.434171ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:49.394316054Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 610.804903ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:49.999002879Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 1.178315673s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:50.107471732Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 1.398100062s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:50.143647703Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 830.811983ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:50.276181025Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 1.049417893s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:50.618907248Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 1.126838966s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:50.732989992Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:50.776705643Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:07:50.777362107Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"bfd19ee9-52b1-4d31-b223-ff8bdb14f8ad","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58033","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":590,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 588.401952ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:51.115528098Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 1.01760747s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:51.295405645Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 1.223696881s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:51.553447229Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 1.472569534s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:51.680033072Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 734.87956ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:51.716377007Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 638.314189ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:51.873304606Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 688.150891ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:52.322066223Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 812.136552ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:52.460054954Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 1.155406172s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:52.520326785Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 992.937964ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:52.78592695Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 919.563977ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:53.01812895Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 839.15547ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:53.160276603Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 730.264069ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:53.280677834Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 1.073708384s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:53.685320301Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 891.020902ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:53.787416197Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 505.875474ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:53.946816554Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 738.241885ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:54.011005754Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 1.466933899s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:54.020272889Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 1.399297859s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:54.463130388Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 996.054405ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:54.687873058Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e\") failed: Property not found. retry in 1.038393089s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:54.758329333Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 983.932403ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:54.768470875Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 1.119965972s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:55.498000869Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 898.473788ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:55.692853954Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 1.003766211s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:55.761137986Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 867.172141ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:55.871092827Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 587.938604ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:56.600903497Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 1.385184307s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:56.626669681Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 1.007988853s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:56.748097772Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 577.466629ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:56.773930387Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 1.258406012s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:57.204119144Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 854.689973ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:57.363584013Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 932.347224ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:57.766450105Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 1.300694016s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:58.021454209Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 638.176911ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:58.22767845Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 672.093438ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:58.472270586Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 1.36979985s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:58.710938794Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 1.245272247s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:58.875024006Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 1.338248262s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:59.15411625Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 857.146472ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:07:59.331509836Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 1.333455842s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:00.089567579Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 832.381786ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:00.129500265Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 1.205472086s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:00.197337958Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 1.250029271s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:00.500866889Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 1.408693261s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:00.970991542Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 721.024447ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:01.412073265Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 943.022815ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:01.431783548Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 1.310614611s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:01.761023032Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 1.166113915s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:02.14260456Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 973.826299ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:02.383331978Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 974.039846ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:02.391538072Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 1.394816146s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:03.079931537Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 770.925159ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:03.316812369Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 1.46536834s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:03.36601259Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 1.419437928s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:03.37354911Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e\") failed: Property not found. retry in 1.041590593s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:04.095358671Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b\") failed: Property not found. retry in 611.546718ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:04.483440207Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0\") failed: Property not found. retry in 1.142208723s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:04.802154932Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d\") failed: Property not found. retry in 1.206018926s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:04.839739041Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:06.071893886Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:06.071996248Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:06.072014398Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:06.072023418Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:06.072033838Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:06.072051549Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:06.072553019Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:06.072944807Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"eee17f95-0eeb-4dcd-88be-cea1cb11887b","remote_addr":"[fd00:1122:3344:101::1]:61545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":22547458,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:06.10623582Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:49560"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:06.139374721Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3165"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:16.310679371Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3183"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:20.81798122Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64504"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:20.818834688Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"41be91b1-f309-4d66-910f-2df9acf54d3b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64504","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":374,"response_code":"200"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.212653178Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3206"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.213195419Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"ac90b76c-777e-4432-b042-76c3e7faf2bf","remote_addr":"[fd00:1122:3344:101::1]:49560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":21107421,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name(\"ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1999ec12-789c-40c8-83be-a68bad0fad69, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 052b5be3-98fd-4823-9614-96d5055e333c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a0570de6-fbf4-4073-9e84-a6e01264b6a9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: b2eaae1a-392a-411d-b8cc-0dff26ddef0c, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9324d491-c0d2-436b-aafc-efea5edebe21, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 7d74b4bc-c408-4088-9473-a933f16aef97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ad7ed868-02d0-452d-8c54-23c7ef814af4, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c22ce43b-b4c5-453b-9120-505102876329, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 1b32292f-6190-435a-87d8-7fdbe1a65f21, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 661755cb-0b27-4b29-a851-a3a45274279d, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 83fba088-495e-4423-abe6-b30a70233d4c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: c7373a32-5911-4b61-b967-b9351fe9441e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 475bc82c-0c96-4f51-8fdf-e66178c8f3fa, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cad70807-3779-4dbd-b5e8-a3a0e7563ca7, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b1234f77-5c34-4c96-bce0-c457facfb566, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3773d36c-8002-400f-9f73-719a40bd0f35, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.25588457Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.256319518Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:56765"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.257155685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.257166855Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.280753396Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.280817897Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.300206243Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.300275635Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.319567719Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.31962562Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.338529157Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.338588609Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.356483685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.356539326Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.374442623Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.374507185Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.39283244Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.392895451Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.41038985Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.410445501Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1999ec12-789c-40c8-83be-a68bad0fad69, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.564580394Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1999ec12-789c-40c8-83be-a68bad0fad69, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.564643995Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2eaae1a-392a-411d-b8cc-0dff26ddef0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.691497854Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b2eaae1a-392a-411d-b8cc-0dff26ddef0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.691560115Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c22ce43b-b4c5-453b-9120-505102876329, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.810250212Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c22ce43b-b4c5-453b-9120-505102876329, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.810314683Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b32292f-6190-435a-87d8-7fdbe1a65f21, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.937738963Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1b32292f-6190-435a-87d8-7fdbe1a65f21, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:27.937805065Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 661755cb-0b27-4b29-a851-a3a45274279d, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.05442701Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 661755cb-0b27-4b29-a851-a3a45274279d, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.054505151Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83fba088-495e-4423-abe6-b30a70233d4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.178181137Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 83fba088-495e-4423-abe6-b30a70233d4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.178248469Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7373a32-5911-4b61-b967-b9351fe9441e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.29320776Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c7373a32-5911-4b61-b967-b9351fe9441e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.293288002Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 475bc82c-0c96-4f51-8fdf-e66178c8f3fa, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.421977637Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 475bc82c-0c96-4f51-8fdf-e66178c8f3fa, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.422048869Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cad70807-3779-4dbd-b5e8-a3a0e7563ca7, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.56354465Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cad70807-3779-4dbd-b5e8-a3a0e7563ca7, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.563619131Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1234f77-5c34-4c96-bce0-c457facfb566, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.680576633Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b1234f77-5c34-4c96-bce0-c457facfb566, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.680639614Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3773d36c-8002-400f-9f73-719a40bd0f35, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.797236509Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3773d36c-8002-400f-9f73-719a40bd0f35, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.79730058Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.889832905Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.889919326Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.890230243Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:28.890556219Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.015399798Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.087289661Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.08770289Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.087742121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.087770791Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.087802802Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.087824942Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.087862543Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.089055137Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.101866452Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.278402222Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.278911152Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.278930892Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.278943993Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.278948542Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.278953043Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.280111066Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.29236778Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.292415221Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.293232047Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.298737227Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.304244127Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.30944047Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.314518772Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.319167294Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.324166844Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.467081793Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.471630724Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.475981741Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.480386418Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.484925219Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.489626293Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.495453899Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.50050457Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.505998689Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.511243634Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.516432987Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.521945417Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.528204822Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.533179531Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.540034708Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.546361384Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.552009846Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.557306802Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.564015736Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.569436274Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.573849332Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.6148934Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.653076731Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.696458836Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.737887742Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.775842699Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.777429931Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.777446161Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.836943327Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", _ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 201, 33]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.837010198Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.839517438Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.839534339Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.895811341Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", _ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 245, 172]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.895885222Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.899204518Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.899224789Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.965310416Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", _ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 134, 173]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.965365107Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.967771115Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:29.967787216Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.027989036Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", _ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 161, 144]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.028050247Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.031341143Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.031359943Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.100158875Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", _ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 210, 225]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.100229456Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"external_dns"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.100796227Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.100807468Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.101075603Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.101097684Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.103502261Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.106998131Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.135317076Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.137779115Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.137838426Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.137852356Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.138026672Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.13803548Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.138200963Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.138215773Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.166445726Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.16967372Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.172077528Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.172097269Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.172120399Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.174402545Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.17516719Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.183808842Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.269777936Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.276241555Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.281887108Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.287857987Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.294460079Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.313018629Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.321249313Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:30.327934966Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.024040206Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.024098168Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.07087455Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.071086904Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.253624844Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.25394277Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.299325975Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.299429827Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.300835245Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.301426927Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.301685122Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.301699172Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.312303644Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.319568618Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.32118159Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.321225182Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.328868324Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.335193Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.342533946Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:33.350398003Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 32.037055ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.438120441Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 58.985644ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.449428917Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 148.616588ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.485615628Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 123.414164ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.544819379Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 65.30659ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.565150634Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 66.841515ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.608179452Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 58.546855ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.623910205Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 33.938378ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.638774112Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 27.639874ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.65423205Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 27.798497ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.674164327Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 134.071274ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.726986351Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 212.540069ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.764875666Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 65.091074ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.789009627Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 75.815755ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.809432184Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 158.685216ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.841241028Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 102.345996ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.875359779Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 60.899076ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.904985159Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 291.353325ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.94312327Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 70.552371ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:33.979436484Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 462.366311ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:34.119420584Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 120.653593ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.472096531Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 105.619685ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.61449627Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 71.314231ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.634877006Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 324.899676ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.650357515Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 117.025679ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.670388654Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 140.872184ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.683793792Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 225.357695ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.703561596Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 224.873684ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.71679051Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 144.293151ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.746483472Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:38.754385739Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 647.349496ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.768429159Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 227.893795ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.782557291Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 481.891782ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.797507319Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 142.19313ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.813222452Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 185.299599ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.828097249Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 235.867784ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.841596528Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:38.84173081Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 256.724412ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.934304686Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 248.783264ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.949499339Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 440.85836ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.963999468Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 403.873281ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.977378335Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 738.51122ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:38.991335163Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:39.009119878Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:39.009689579Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 432.415927ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.041765798Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 264.695057ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.056520413Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.064335836s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.097114572Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:39.09753657Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 48.812742ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.183520585Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 287.084777ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.207525063Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.163706339s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.222255087Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 90.02551ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.245325507Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 846.84572ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.296255262Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 50.751462ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.308785792Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 491.901484ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.337749179Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 172.960439ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.35132466Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 66.672119ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.374504722Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 69.593638ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.385736136Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 669.219575ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.401711045Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.363266645s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.418130732Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 755.75501ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.437229543Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 172.308928ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.454957956Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 69.565957ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.46867889Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 825.547378ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.490407613Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.162170348s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.509709578Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 400.990416ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.541712716Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 299.236264ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.555646624Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 316.75412ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.642598987Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.359952451s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.746358636Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.458899489s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.844778978Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 227.941601ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.870432569Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 777.347576ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.956138218Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 664.954791ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:39.973639027Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:39.9838321Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:39.984435052Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:40.030331127Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.192675534s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.084501087Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 978.274775ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.111352333Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 716.909594ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.156909041Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.06895817s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.175357529Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 630.424203ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.206057111Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 29.931328ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.220607811Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 104.372312ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.261016766Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 50.373676ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.275937234Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 1.196448887s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.32839392Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 99.395502ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.33796568Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 242.171169ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.37757198Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.047366502s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.39816706Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 275.963753ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:40.4493207Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.415182928s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.029561498Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.03392668s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.068120007Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 439.923649ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.0998753Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 786.439681ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.125983131Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 508.470683ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.156434178Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.245129249s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.182819014Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 585.926008ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.22427165Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 543.196686ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.24133981Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 465.148139ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.261303748Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.08587875s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.276887009Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.331737684s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.290527321Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 733.419775ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.303284836Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.466912347s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.317984538Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 691.395736ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.459391347Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 852.066953ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.538128027Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 509.327485ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.55333415Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.113447414s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.748009051Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 483.374799ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.76498871Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 1.124916293s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.801866735Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 783.103504ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.822950356Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.165572766s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:41.925363567Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.21402433s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.05092407Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.346707043s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.077201554Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 876.022664ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.114446507Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.122641688s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.164959044Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.075517871s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.262239103Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.404632889s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.376704045Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 1.238304288s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.402717164Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 968.663265ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.439230732Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 836.222025ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.459485086Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 661.824886ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.619252991Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 804.198395ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.634135528Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.24029257s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.798466794Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 647.836098ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.87402841Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 586.535618ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:42.940128528Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 731.859577ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.00444104Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 999.986259ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.105162988Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.352549287s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.275493124Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.31924644s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.302398131Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 613.384286ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.310187106Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.035815835s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.31792688Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.403741631s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.346694494Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.311777224s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.417262471Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.252460213s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.433723399Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.218248471s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.449255099Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 516.132977ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.531547849Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 1.238506056s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.539644281Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 579.242393ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.653758416Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.491933462s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.747362922Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 966.629601ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.792544972Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.15157419s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:43.936689256Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.25321477s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.051059466Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.022736949s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.062623857Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.287148665s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.117390519Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 910.298532ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.245766298Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.497681785s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.362780201Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:44.491008317Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:44.496205841Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:44.583388109Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 646.765761ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.631408896Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 696.711379ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.64063648Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 41.674161ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.649392255Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.240957212s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.677447064Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 580.4933ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.696576556Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 58.159465ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.704571625Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 52.068417ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.714926101Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 908.053983ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.738248837Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 793.541867ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.760351637Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 533.734155ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.768408508Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 58.63086ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.775266175Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 193.482351ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.782331965Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 1.454076534s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.790914136Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 171.777682ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.843040996Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 242.723566ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:44.987334532Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 324.467772ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.027129716Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 26.168566ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.041519843Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 129.066634ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.076726525Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 884.460449ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.093613461Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.011257001s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.101214843Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 944.733709ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.16575478Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 164.179575ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.213797807Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 438.404344ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.238449049Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.093118s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.249944038Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.320219305s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.285824113Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.128620754s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.294455566Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 512.584886ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.311479275Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 750.407367ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.319574256Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.456865159s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.345456242Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 708.064627ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.359798588Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 403.2817ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.385877848Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 637.318844ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.412454147Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 664.972731ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.564304645Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.232217871s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.656078624Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.17096783s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.686635444Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 506.39045ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.798058625Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.345311176s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.833249877Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 930.578863ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.8690235Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 912.657253ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.926552197Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.119081535s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:45.985686116Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.167183493s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.060838794Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.196655613s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.077902794Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 975.091503ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.086027406Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 978.946251ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.118895792Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.266943796s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.126436962Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 871.707861ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.240094378Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 926.087236ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.256099067Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 976.919801ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.315810637Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.220380428s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.352196923Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 727.797405ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.432550095Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 862.26684ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.615160015Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 525.176712ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.809407968Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 657.842738ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.818151632Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 554.903331ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.848550258Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.263336291s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.866811122Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 514.796858ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:46.896526905Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.125647407s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.070451902Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 510.561069ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.118927659Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 711.359446ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.127121772Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.456669826s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.135595311Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.330662075s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.168499497Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.407771908s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.187565227Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 1.413720748s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.195663789Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 555.264792ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.236945701Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 878.347316ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.283109352Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.28751179s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.302367966Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 557.647324ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.344573307Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 762.064153ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.401918321Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 664.365502ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.412049752Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.174762579s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.420160484Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.325515441s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.485400025Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 550.818773ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.492930515Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 825.755843ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.581494501Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 629.415335ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.637994917Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 554.4028ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.802257062Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.421420534s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.847188338Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 611.313776ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:47.911124942Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.406251761s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.053451599Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 659.19332ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.085661252Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.218509117s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.139143798Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.195632583s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.170483583Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 686.590018ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.178339259Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 952.516529ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.20596826Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 1.415529554s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.276512677Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 690.607339ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.365353188Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 880.019434ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.416081869Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.171104522s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.508550882Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 912.6091ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.53151315Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.271238991s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.5997087Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.357407385s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.607308541Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.103327978s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.614847171Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 664.970508ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.622233369Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 1.119827941s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.631442272Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 960.420487ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.755759001Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 551.124167ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.82091143Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 796.927866ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:48.875148091Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.073129276s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.067243781Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 505.205407ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.167485519Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 637.048388ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.278173896Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.18983857s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.297022611Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 549.395458ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.304921689Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.283506603s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.366223571Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.282031626s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.375259281Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 971.925544ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.382828112Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 883.116128ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.454155714Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.462252034s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.480751804Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.211835745s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.698347632Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.067527708s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.714651727Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.129035726s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.725185617Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 808.885606ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.735818319Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.470956289s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.746178256Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.117961295s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.756792047Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 825.977266ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.766640964Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 712.137249ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.864113207Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 689.572808ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.880074035Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.096961989s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.924738646Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.138029526s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:49.97563187Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 809.008992ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.150504586Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.313491542s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.346131117Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.136788863s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.365159526Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.086337037s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.496578146Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 660.208732ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.553121803Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 849.484057ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.57853699Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.398031463s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.586272484Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 915.07971ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.602186291Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.050972694s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.659560095Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.1774151s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.667524934Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 842.456863ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.790564907Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.493312684s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.867631623Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 812.288354ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.888740194Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 540.079037ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.921692811Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 788.964485ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.954285531Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 805.921296ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:50.972335501Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 845.893534ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.035314406Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.462029665s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.123689538Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 1.419708587s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.221539249Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.187179437s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.229646311Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:51.259031606Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59693"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:51.327007172Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"ca81bf78-fd66-4ce7-b916-9235890f01cd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59693","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":67677,"response_code":"200"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 560.382017ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.439273119Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 864.923512ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.471811808Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.236499634s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.51156912Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 532.650073ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.525751733Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.393607539s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.592725918Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.237322261s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.642835768Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 762.324382ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.66854385Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 680.292285ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.711239651Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:51.711612468Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.28406776s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.720814942Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:08:51.720985755Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.009690136s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.754232488Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 639.55965ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.789782227Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.341612584s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.855271683Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 577.94168ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.892036896Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 42.802677ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.90382592Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 129.12156ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.954187365Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 971.549856ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:51.993780244Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.351075044s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.007944646Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 671.326307ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.070439432Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 234.861531ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.105685985Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 703.21809ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.347393384Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 468.319824ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.356937884Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.344721238s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.370499674Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.262406989s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.400948221Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 721.734838ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.426211835Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.043459537s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.438789616Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 846.175862ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.446901247Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.377701119s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.479303783Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 712.007193ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.595569511Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 680.257857ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.65018405Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 940.77057ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.751607082Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.27168745s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.75953641Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.258844318s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.773405776Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 533.223737ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.835467244Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.401508479s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.890236186Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.220924152s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.974469735Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 926.094085ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:52.995822511Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.13912863s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.015037254Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 638.060724ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.059319857Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.124572671s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.157879652Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 731.69469ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.206906459Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 815.453491ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.303466584Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 926.171457ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.317244139Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 618.13214ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.340265238Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.061827103s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.368700724Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 1.127909364s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.377259815Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.066601201s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.491898471Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.279491731s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.673587303Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 919.616237ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.701929768Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.427321661s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.708787704Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 697.814523ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.722998468Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 827.460254ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.867035039Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 546.192712ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.932715189Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 579.332961ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.946646526Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 824.909457ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:53.967318819Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.067066881s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.041974927Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 549.23633ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.050672811Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.025076491s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.129676275Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 936.795058ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.163796166Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.093579075s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.203846084Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 841.599867ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.251692028Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 772.527694ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.292354179Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.014111458s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.301838998Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.091046816s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.430025993Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.463630612s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.439126634Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.335429127s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.490710883Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 1.439118529s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.514935976Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.240379484s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.536113518Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 835.558276ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.567726038Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 618.113081ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.609104253Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 770.723776ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.630057711Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 593.218857ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.703120878Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 1.068891066s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.801265464Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 608.66535ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:54.963210033Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 768.98734ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.075372499Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.178317233s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.103497789Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 792.105558ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.112220313Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.402665193s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.121174062Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 946.982878ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.1456549Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.264092035s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.163605428Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.428385712s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.237341228Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.258354848s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.30617685Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 545.237952ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.315395924Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.339844863s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.325967665Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 924.365852ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.410094762Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.04152536s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.417979559Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 810.018964ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.531988102Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.343156255s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.581803115Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.145388908s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.787246151Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 725.946888ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.835690867Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.05565334s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.852712036Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 631.030074ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.869404869Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 1.485622019s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.87896858Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.444486542s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.910786504Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.478924782s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.917659941Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 1.170973797s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:55.962908563Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.469003904s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.104073957Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 819.385509ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.29283432Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 688.439219ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.343262336Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.181737716s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.350653933Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.248426705s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.437821321Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.132354256s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.467984092Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.067736044s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.509732494Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.069769471s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.533980578Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.043468524s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.570980325Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.445583964s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.579335612Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 764.956216ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.677317155Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.400482338s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.686849205Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 915.696845ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.920745018Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.347402361s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.935986832Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 936.418016ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:56.945100664Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 1.314007549s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.0447453Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 675.722545ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.123034411Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 1.168711966s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.144822096Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.185685346s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.366191738Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 1.463762458s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.375329761Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.132983173s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.406863479Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.128273508s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.451566231Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 722.110367ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.543525944Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 503.325944ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.584782786Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 614.53544ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.594046551Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.062237672s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.609352806Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 937.807802ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.617782134Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.151744899s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.628237783Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 941.741419ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.698198388Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.016525269s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.813132349Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 530.121329ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.848199648Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.363546025s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:57.893344378Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 595.787671ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.037489182Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.000877506s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.099859575Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.393279558s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.109911325Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 861.429873ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.21947984Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.158404023s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.276304803Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 894.149996ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.292969075Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 611.887835ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.320752929Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 735.753914ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.366653494Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 502.582084ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.385969789Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.351567799s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.550149432Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.371609268s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.563029279Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.416807131s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.570942147Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.340721633s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.588454806Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 832.876155ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.6453481Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 800.864172ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.655848389Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 597.106997ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.682386209Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 971.399702ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.791366911Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 502.842422ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.839187404Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 1.14468202s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.849314786Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.399253314s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.898400675Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 1.298255641s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:58.942591576Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.129591932s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.090561816Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.256116721s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.108683527Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 851.413204ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.116983033Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 516.229556ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.197409336Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 945.740837ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.267705727Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.48715812s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.28889191Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 954.006743ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.352506168Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.063419561s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.447509342Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.060437171s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.471076232Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 553.650676ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.491786854Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.448691114s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.514264273Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 512.210787ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.724383422Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 741.469426ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.773553282Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 632.484571ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.911998552Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 973.86899ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.940539741Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.367301682s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.949286955Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 850.737186ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.978050929Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.056586858s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:08:59.997640069Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 836.239558ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.005514016Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.248055596s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.057566144Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.085733361s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.225224947Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.422388611s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.234598623Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.068851741s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.247208465Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 819.509842ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.256036681Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.379501266s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.3086675Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.356517467s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.320941435Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.161211546s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.37941011Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 616.295824ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.523477313Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.45473491s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.535311709Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 1.419009732s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.546115734Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.406072261s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.556105663Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.476194001s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.792950755Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 1.307183543s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.843210997Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 942.709477ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.856257767Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 936.924028ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.934096259Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 575.619461ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:00.982558055Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.289598451s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.071701492Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 1.322302485s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.094106279Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 643.121869ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.164285888Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 658.044454ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.325233797Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.152705886s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.342280486Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.271520549s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.364806785Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 540.886917ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.381737023Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.226755764s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.58075964Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.284423305s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.600737019Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 802.136681ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.679633531Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 527.144345ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.704914275Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.033165778s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.729167659Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 1.114733098s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.821395128Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.24833896s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.840798264Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.157863032s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.888610558Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.478129387s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.940988402Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.345218971s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:01.985962489Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 975.813252ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.00760213Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.495521379s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.02663878Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 873.988529ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.049611378Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 889.743322ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.168554819Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 870.486579ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.249157416Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 580.441928ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.286835427Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.490371861s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.375510575Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 1.16333641s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.432268226Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 848.137997ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.494510007Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 866.673856ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.507768861Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 1.323491373s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.652921205Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.210597425s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.775347916Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 1.145185094s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.820500876Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.4166746s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.893461301Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.189810403s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.904394879Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 819.563849ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.933730213Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 636.3715ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.945038009Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 824.014571ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:02.993010385Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 700.650828ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.058717655Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 1.131178698s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.069397988Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 1.49948274s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.100367035Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 542.816453ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.129374824Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3\") failed: Property not found. retry in 1.329223086s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.34389553Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 538.253504ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.35491757Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 639.604301ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.386211714Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 613.93729ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.43016212Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 584.870702ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.534194154Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21\") failed: Property not found. retry in 1.331149377s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.5946861Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 1.076352977s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.606814032Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97\") failed: Property not found. retry in 1.047205832s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.684291756Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21\") failed: Property not found. retry in 1.418893603s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.764930714Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9\") failed: Property not found. retry in 1.133398827s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.774356672Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566\") failed: Property not found. retry in 903.20267ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.828108714Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa\") failed: Property not found. retry in 1.411042535s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.878335825Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c\") failed: Property not found. retry in 1.403151427s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.904351704Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329\") failed: Property not found. retry in 880.907063ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.977657595Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5\") failed: Property not found. retry in 859.530033ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:03.986994701Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e\") failed: Property not found. retry in 1.279111742s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:04.007358507Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c\") failed: Property not found. retry in 1.187765128s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:04.038898286Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\") failed: Property not found. retry in 1.344120743s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:04.060403275Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d\") failed: Property not found. retry in 1.417818539s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:04.1057926Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69\") failed: Property not found. retry in 1.435870404s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:04.129365Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7\") failed: Property not found. retry in 745.730885ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:04.211457706Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35\") failed: Property not found. retry in 1.287240767s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:04.321424038Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4\") failed: Property not found. retry in 505.122831ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:04.611580023Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c\") failed: Property not found. retry in 744.156685ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:04.697667279Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::c exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:04.727989114Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::c, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:04.728046505Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::c exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:05.301210692Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:05.301290913Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:09.017876887Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1971"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:09.017934818Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Ensuring address fd00:1122:3344:101::b exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:14.92991251Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:14.929982881Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:15.482574908Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:15.482652909Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:15.699608155Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1971"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:15.699677606Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:21.36402676Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51490"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:21.364139833Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"9de83871-36f1-4c83-8ce4-96b487c77122","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51490","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":80,"response_code":"200"} {"msg":"Ensuring address fd00:1122:3344:101::9 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:22.066830902Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::9, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:22.066891573Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::9 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:22.622310766Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:22.622382927Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up external-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:22.833522947Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2084"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:22.833594638Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Ensuring address fd00:1122:3344:101::a exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:28.365837969Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::a, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:28.36590088Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:28.921997987Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"243.107168304s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:28.923191231Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"243.110558332s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:28.923401215Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"243.11048233s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Ensuring address fd00:1122:3344:101::a exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:28.924400845Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:28.924412105Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:29.01093837Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"243.198925594s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:29.161766977Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1971"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:29.161801268Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:29.740851722Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"243.928712323s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.518531321Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"245.704005845s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.590988116Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"235.468961749s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.591138759Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"22.128172115s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.591282682Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"245.777474069s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.59218682Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"245.777723245s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.592267331Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"245.779839987s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.592347423Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"245.780618673s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.592398084Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"245.781153303s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.592461005Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"245.778965229s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.600792281Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"245.788596312s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.601032046Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"245.789557571s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.601321402Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"245.78902958s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.601572287Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"245.788486839s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.601633618Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"245.787821156s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.601677639Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"245.788456579s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.60172504Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"245.79104496s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:31.602268841Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"245.788125652s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:32.341625551Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"246.529746217s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:32.471192744Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"246.659217608s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:32.843460355Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"247.031357377s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:33.150849083Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"247.338089912s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:33.151050247Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"247.33848847s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:33.151106759Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"247.336826988s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:33.485472854Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:33.48827446Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"237.366267673s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:34.652337767Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"248.840044125s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:34.716862154Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"248.902364957s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:34.956892099Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"249.143398493s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:35.281899378Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"249.468998364s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:35.647785682Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:35.655339773Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"239.533315956s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Ensuring address fd00:1122:3344:101::8 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:35.673612337Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::8, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:35.673684139Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:36.220711835Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"250.409469523s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Ensuring address fd00:1122:3344:101::8 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:36.228097482Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:36.228125972Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up external-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:36.425887155Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2084"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:36.425943916Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:36.450846592Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"250.638649303s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:36.561175402Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"250.749702946s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:36.68597617Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"250.871694179s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:36.847119723Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"251.034567965s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:36.874460168Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"251.063794777s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:36.942831961Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"251.12960956s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:37.076097097Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"251.263368836s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:37.372730001Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"251.560641193s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:37.458163854Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"251.646448774s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:37.57329242Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"251.760878795s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:37.645418847Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"251.83235657s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:37.938539741Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"252.124741669s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:37.95704185Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"252.142921802s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:38.226853639Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"252.413957595s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:38.486389473Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"252.671902027s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:38.523546624Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"252.709880244s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:38.606595949Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:38.608895095Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"242.486902318s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:38.721337987Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"252.909488274s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:38.886047721Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"253.072084435s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:38.972039645Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"253.16058351s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:39.157956312Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"253.345772492s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:39.90626535Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"254.093984668s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:40.034353293Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"254.221804846s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.087111271Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"255.27437582s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.141334272Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"255.329633642s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.223040231Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"255.410964333s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.266367985Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"255.455153374s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.356721726Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"255.543516436s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.605905904Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.605976165Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.605996036Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606006116Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606021016Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606028356Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606034466Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606060317Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606068137Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606082587Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606088498Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606096538Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606106658Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606112798Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606122608Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606127958Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606133118Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606678559Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.606687019Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.607423864Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.609817342Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.611573877Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"63c68e8c-a816-4064-ac0c-f9e4459aa3e2","remote_addr":"[fd00:1122:3344:101::1]:56765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":74355532,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.616681429Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1059"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.617676109Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.617823951Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.621183948Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.621200389Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:551"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bgp: [],\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.655637655Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fdf13508-1e61-48db-a883-3beaba642d42/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.656730117Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.658682306Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"255.845630749s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.691120843Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"255.878243868s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.718709233Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"255.905227206s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fdf13508-1e61-48db-a883-3beaba642d42/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.723187932Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.789426422Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"255.975162281s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fdf13508-1e61-48db-a883-3beaba642d42/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.811643496Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:41.875708363Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.87756196Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"245.755577603s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.898897115Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"256.08694675s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:41.907880384Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"256.09548775s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:42.004718695Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"256.191076715s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fdf13508-1e61-48db-a883-3beaba642d42/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:42.015960789Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fdf13508-1e61-48db-a883-3beaba642d42/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:42.140352309Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"48b066d6-729e-4f5e-9200-29775e06db75\", \"content-length\": \"160\", \"date\": \"Thu, 15 Feb 2024 00:09:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: ip-pool with name \\\"oxide-service-pool\\\"\", request_id: \"48b066d6-729e-4f5e-9200-29775e06db75\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:42.346931387Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c008e4f4-baf6-4278-a28f-146a5ba01809\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:09:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c008e4f4-baf6-4278-a28f-146a5ba01809\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:42.539240501Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3da3b0eb-3e11-4787-b314-7bc1c91a0a8e\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:09:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3da3b0eb-3e11-4787-b314-7bc1c91a0a8e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:42.76290842Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:42.78446681Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"256.970340071s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"488d3754-c349-4e05-81bc-db71ab118b6a\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 00:09:42 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"488d3754-c349-4e05-81bc-db71ab118b6a\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:42.81962196Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"257.007343548s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"995a0562-3330-4f9d-ac6d-096b6b955502\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 00:09:42 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"c76945c0-20aa-4b87-817a-70fea9ec6d5a\\\\\\\"\\\", request_id: \\\"995a0562-3330-4f9d-ac6d-096b6b955502\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"de1381fe-b1bb-4ff1-8ddb-74e776a987de\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:09:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"de1381fe-b1bb-4ff1-8ddb-74e776a987de\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:43.175101438Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:43.193043945Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"257.380867296s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"0cf574e6-82c1-439d-9583-fec3173a086b\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 00:09:42 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"c76945c0-20aa-4b87-817a-70fea9ec6d5a\\\\\\\"\\\", request_id: \\\"0cf574e6-82c1-439d-9583-fec3173a086b\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0b58851e-db9c-4959-902a-de749c4215ea\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:09:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0b58851e-db9c-4959-902a-de749c4215ea\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:43.804473205Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:43.864654234Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"258.053982694s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"39037b30-5578-45ce-92d2-481e40fc4207\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 00:09:43 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"c76945c0-20aa-4b87-817a-70fea9ec6d5a\\\\\\\"\\\", request_id: \\\"39037b30-5578-45ce-92d2-481e40fc4207\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8d975738-a501-4363-84fb-70a056e1655f\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 00:09:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8d975738-a501-4363-84fb-70a056e1655f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:44.402224311Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:44.680335206Z","hostname":"sock","pid":1671,"sled_id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:44.713875515Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"c76945c0-20aa-4b87-817a-70fea9ec6d5a"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6023bbef-7419-4a1c-ae82-36a575f73bf8\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 00:09:45 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"6023bbef-7419-4a1c-ae82-36a575f73bf8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:45.468103751Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:45.794662451Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"259.980867199s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"7137800e-b526-497c-8bb8-c7bbe823bc8f\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 00:09:45 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"7137800e-b526-497c-8bb8-c7bbe823bc8f\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"06baa4b1-33bf-472f-97e4-da3cef764a6a\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 00:09:46 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"06baa4b1-33bf-472f-97e4-da3cef764a6a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:46.538568652Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:46.712364086Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"260.899158936s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"a5cb1ef0-5ae2-4e0a-95b2-c0858082aef1\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 00:09:46 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"a5cb1ef0-5ae2-4e0a-95b2-c0858082aef1\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:47.275952862Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"261.46322478s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"8290b76d-dce6-42f8-b1fd-eb720cb81574\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 00:09:46 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"8290b76d-dce6-42f8-b1fd-eb720cb81574\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cbeee3b5-8366-4cc9-a661-4dd320b9d4d2\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 00:09:46 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"cbeee3b5-8366-4cc9-a661-4dd320b9d4d2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:47.448541173Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:48.962498716Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"263.148862896s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"ee3a10ca-929c-428e-9698-4bbbc3d7c783\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 00:09:48 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"ee3a10ca-929c-428e-9698-4bbbc3d7c783\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9840426-303d-4536-a414-50265f70e4b6\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 00:09:48 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"f9840426-303d-4536-a414-50265f70e4b6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:49.343909589Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:49.507148194Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:49.508133643Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:49.508141473Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:50.15805518Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:50.158078911Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:50.158085591Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:09:51.007231889Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"265.193448957s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"ea1d6b72-6fd8-4d13-83ca-ac668d0b4f2d\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 00:09:50 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"ea1d6b72-6fd8-4d13-83ca-ac668d0b4f2d\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d026365f-d129-4c39-b6c7-949712a54035\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 00:09:50 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"d026365f-d129-4c39-b6c7-949712a54035\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:51.120066679Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:51.404851066Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48586"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:51.4050782Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"c6c4b870-989d-474f-8e31-f907101a9a73","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48586","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:52.374094809Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:52.374120209Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:52.374125739Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:52.37413009Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:52.37414303Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"38042dd0-767f-4b6d-b6fb-58284bace98c\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 00:09:51 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"38042dd0-767f-4b6d-b6fb-58284bace98c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:52.887698848Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:53.649121048Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:53.649143978Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:53.649150848Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:53.649155418Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:53.649160248Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5fb5c751-5454-4a3a-9838-ccee04e3c3c3\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 00:09:54 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"5fb5c751-5454-4a3a-9838-ccee04e3c3c3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:54.335870459Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"169a15af-a32d-4aec-b698-ccfc5bbab05b\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 00:09:56 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"169a15af-a32d-4aec-b698-ccfc5bbab05b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:56.155186098Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a0ebe89b-6679-4017-be0b-96bfd4b2aa7c\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 00:09:57 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"a0ebe89b-6679-4017-be0b-96bfd4b2aa7c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:57.967973037Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:09:58.87701978Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d7b0f5e9-6404-4f29-935e-3a7b07276172\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 00:10:02 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"d7b0f5e9-6404-4f29-935e-3a7b07276172\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:02.49864187Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:02.939545429Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:02.93956619Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:02.93957234Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:02.93957699Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:02.93958186Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.568063067Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:43334"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.572072577Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.57222997Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.57225045Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.57225731Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.57226691Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.572283061Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.572289151Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.572305141Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d0b8872f-8b27-4319-aee6-df970101cccb","remote_addr":"[fd00:1122:3344:101::b]:43334","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":4167,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7814c8ea-eeee-48a3-9246-d5edd5445256\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:10:05 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"7814c8ea-eeee-48a3-9246-d5edd5445256\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.692718312Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.818076921Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:51317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.818612682Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"b52539cf-f857-4a56-b206-928738e5ae0f","remote_addr":"[fd00:1122:3344:101::b]:51317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":512,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.819385879Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.819670653Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.820281095Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:06.820481309Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"c53f0300-662b-42a4-b7c8-0bb706e58f5d","remote_addr":"[fd00:1122:3344:101::b]:51317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1564,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:09.939018546Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41520"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:09.939832477Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:09.939877908Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:09.939899728Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:09.939907668Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:09.939930239Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:09.939937119Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:09.939944209Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:09.939950809Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"92c242dd-6f18-4d68-bb3d-61bcbf786b8a","remote_addr":"[fd00:1122:3344:101::b]:41520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":866,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"73ca300f-ef75-49af-91d4-9f7aacbe8d87\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:10:09 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"73ca300f-ef75-49af-91d4-9f7aacbe8d87\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:10.017533397Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:10.100117221Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:38011"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:10.100183352Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"b26a7c70-b0bf-458e-9d01-5e96891638c3","remote_addr":"[fd00:1122:3344:101::b]:38011","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":44,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:10.100932676Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:10.100947807Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:10.101135686Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:10.101332771Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"c2dbee7f-c60f-42ae-aac7-d38a7976578d","remote_addr":"[fd00:1122:3344:101::b]:38011","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1046,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.725710885Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:38638"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.7259359Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.726000271Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.726029881Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.726060512Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.726070602Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.726079782Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.726089232Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.726110383Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"08e39ee5-86bd-4542-8327-7f51d6ddec7f","remote_addr":"[fd00:1122:3344:101::b]:38638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":369,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6503f5ee-42b5-4ac2-8b42-37568eeda8d1\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:10:15 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"6503f5ee-42b5-4ac2-8b42-37568eeda8d1\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.773986667Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.84834609Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48136"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.848423691Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"1712d548-e861-402b-a954-c7309b4bbef2","remote_addr":"[fd00:1122:3344:101::b]:48136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":47,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.848637806Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.848664986Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.84884734Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:15.849027614Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"a3f786ad-d1df-4912-98ee-c3afa222fece","remote_addr":"[fd00:1122:3344:101::b]:48136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":429,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.43967007Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42907"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.440373973Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"0b76616a-0a29-4f88-9b56-515fc3c8855a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42907","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":678,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.575690281Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:40551"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.575956267Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.576016198Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.576053929Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.576062849Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.576069829Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.576078829Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.576089119Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.57612708Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"de22271c-7d98-4ff7-8672-9499529fef03","remote_addr":"[fd00:1122:3344:101::b]:40551","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":359,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dd08b6a9-23fe-4050-95b8-428958d87a50\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:10:21 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"dd08b6a9-23fe-4050-95b8-428958d87a50\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.628923443Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.704372047Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52485"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.704478079Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"36757c98-eb5f-4e94-9670-b0ce1fdf637d","remote_addr":"[fd00:1122:3344:101::b]:52485","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.704735394Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.704769795Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.704967969Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:21.705169023Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"f50c8266-8082-480e-8b61-252b8f88a57b","remote_addr":"[fd00:1122:3344:101::b]:52485","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":474,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:23.846675776Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:45486"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:23.857516862Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"f409ae92-6bd0-40a2-9c73-33ee1cbb5511","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":10809,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:29.88817361Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:58744"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:29.889194111Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:29.889229102Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:29.889260042Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:29.889267342Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:29.889273472Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:29.889278872Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:29.889284423Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:29.889299823Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"11d72972-438d-4a40-81f4-e585cc3c0c0b","remote_addr":"[fd00:1122:3344:101::b]:58744","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1038,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4e6529c1-6367-4b93-8f51-515e79f3c0d9\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:10:29 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"4e6529c1-6367-4b93-8f51-515e79f3c0d9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:29.941647387Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:30.028958687Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:62667"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:30.029044939Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"6a797ca4-de59-487a-864e-d2e8d783053d","remote_addr":"[fd00:1122:3344:101::b]:62667","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":53,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:30.029223693Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:30.029260083Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:30.029545989Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:30.029863305Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"5ab0e011-284e-4185-a8a1-dc54056c808e","remote_addr":"[fd00:1122:3344:101::b]:62667","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":670,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.842142484Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:38772"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.842276507Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.842305997Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.842332038Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.842481041Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.842485991Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.842493711Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.842500581Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.842527272Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"451b106a-bafd-489b-9153-24c5c868aeb0","remote_addr":"[fd00:1122:3344:101::b]:38772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":355,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"33cb639e-6323-4b22-9137-d1ae0aaa1723\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:10:34 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"33cb639e-6323-4b22-9137-d1ae0aaa1723\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.886654431Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.968065004Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56065"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.968383031Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"811549ae-a53a-4c4b-a4d2-0206d1599292","remote_addr":"[fd00:1122:3344:101::b]:56065","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":31,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.968597825Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.968617035Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.970381611Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:34.970584775Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"9b04e976-02c4-4224-b886-e48538cf9a06","remote_addr":"[fd00:1122:3344:101::b]:56065","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":2012,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.669254106Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42100"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.669477011Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.669534432Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.669575713Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.669585373Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.669592163Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.669600103Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.669612183Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.669651464Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8dbdcba0-56fb-45d1-9120-b1126e35af98","remote_addr":"[fd00:1122:3344:101::b]:42100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":353,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d9957622-f9a2-40d2-9ee7-e4d2e0437e7d\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:10:50 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"d9957622-f9a2-40d2-9ee7-e4d2e0437e7d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.722135771Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.808943751Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:43288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.809563803Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"fa6ebcd8-e57f-47ee-96e1-a7da81a76fdd","remote_addr":"[fd00:1122:3344:101::b]:43288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":599,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.809703266Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.809729097Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.809970182Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:50.810165726Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"f64e7f4e-6335-4380-8c70-4b6b8daf1222","remote_addr":"[fd00:1122:3344:101::b]:43288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":487,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:51.472301426Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47392"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:51.472349927Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"a4bdd733-a263-4a39-8848-b17c8f13d0cd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47392","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":29,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:10:53.846806425Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"cc90d5c9-a2a5-46ce-afc6-0e3f7135bb0a","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":192,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:08.414623881Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.691027648Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:49113"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.691172931Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.691216482Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.691233492Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.691238912Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.691245732Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.691251093Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.691257103Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.691280273Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"eff13b0b-ec06-499a-9224-13b3c608d0e2","remote_addr":"[fd00:1122:3344:101::b]:49113","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":219,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f11afaaa-2890-41e7-ab31-ee7e7236d52a\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:11:09 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"f11afaaa-2890-41e7-ab31-ee7e7236d52a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.741582986Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.836414537Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41816"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.836528299Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"fbd0ecba-8bd3-40e1-a928-fe218f5eaffb","remote_addr":"[fd00:1122:3344:101::b]:41816","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.836719413Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.836744953Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.836989138Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:09.837168332Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"c86de967-3b47-4982-8a30-71a6c3ac60d7","remote_addr":"[fd00:1122:3344:101::b]:41816","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":469,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:21.504482521Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56873"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:21.504530882Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"a10f58cb-2bd5-48a8-a1ff-4d1a336a46d0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56873","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.506557908Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61567"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.506714141Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.506774752Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.506785142Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.506793222Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.506799883Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.506806753Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.506813513Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.506825903Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6bbc2d5f-cfdb-431d-894b-bcde2ed634a2","remote_addr":"[fd00:1122:3344:101::b]:61567","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":236,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ca661aea-e963-4db5-a5c7-e176becfb8ef\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:11:22 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ca661aea-e963-4db5-a5c7-e176becfb8ef\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.555476333Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.645534939Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52627"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.64560004Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"724baa39-5172-4e89-b65b-a3838dc29114","remote_addr":"[fd00:1122:3344:101::b]:52627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":42,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.645767363Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.645787093Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.646044159Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:22.646232142Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"54a735b0-ae72-4bbd-9c6b-21b0c29cb0c9","remote_addr":"[fd00:1122:3344:101::b]:52627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":499,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:23.846722106Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"68266090-5766-4683-bcf3-73a0f5578067","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.042282496Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:59051"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.042450929Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.04249854Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.042526651Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.042532421Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.042537591Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.042544781Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.042558471Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.042572062Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"20aa09c5-0f5c-46c7-8d8d-96f2a30c56d7","remote_addr":"[fd00:1122:3344:101::b]:59051","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":250,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2eeafcd7-5b60-4118-9c6e-0c64c163aca5\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:11:43 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"2eeafcd7-5b60-4118-9c6e-0c64c163aca5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.093010147Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.179178785Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:45489"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.179277897Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"ef80e368-9afb-42c2-96d0-547feff24e43","remote_addr":"[fd00:1122:3344:101::b]:45489","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.180195755Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.180214546Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.18044989Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:43.180622144Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"a31ea8a8-50ee-4bf2-8856-8b6682aa5c47","remote_addr":"[fd00:1122:3344:101::b]:45489","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1164,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:51.535301374Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45189"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:51.535355165Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"4722b5e1-9233-4d9e-9dc1-9a0b5bf81984","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45189","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":42,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:53.847052481Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"644cf725-eac7-40d8-8096-613cdb48190f","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":177,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.24363129Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48924"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.243774682Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.243825593Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.243845504Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.243852074Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.243858114Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.243864564Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.243870734Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.243895955Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"58aafc7a-bb9a-436d-a7a5-5592e548c950","remote_addr":"[fd00:1122:3344:101::b]:48924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":239,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e3470340-1d7b-43fd-be5e-94d2a3051960\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:11:56 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"e3470340-1d7b-43fd-be5e-94d2a3051960\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.298594125Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.407129269Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.40719028Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"101e1843-1622-41b9-8d6f-58874ade649c","remote_addr":"[fd00:1122:3344:101::b]:56496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":40,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.407385024Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.407405184Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.407618679Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:11:56.407779942Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"89295c3b-e1f0-4a51-8ddc-4b63b6e30d6b","remote_addr":"[fd00:1122:3344:101::b]:56496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":418,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:21.569402156Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51843"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:21.569447057Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"18ded7a9-686c-47d2-b667-d09627ab4631","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51843","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":27,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:23.8463587Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"d90a0aab-2f94-43d1-9e38-83b3e2406e4c","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.766940362Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:64340"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.767120265Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.767162126Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.767188847Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.767195557Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.767200577Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.767205967Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.767212057Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.767234608Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"62ec6d64-e719-426f-976c-8b3f5c6f4062","remote_addr":"[fd00:1122:3344:101::b]:64340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":236,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"42a87db8-118f-419c-8924-6d49b805f453\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:12:24 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"42a87db8-118f-419c-8924-6d49b805f453\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.821588491Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.915158427Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:49781"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.915221488Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"579a7d8c-1fbf-4c41-8bd2-63552b745be2","remote_addr":"[fd00:1122:3344:101::b]:49781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":39,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.915957863Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.915981043Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.916223888Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:24.916393321Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"36eb5752-4ea5-4c51-8e9b-da0341c77cea","remote_addr":"[fd00:1122:3344:101::b]:49781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1048,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:51.605135836Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65003"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:51.605193977Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"f9cd6362-7475-41ef-8979-30b5a63d8502","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65003","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:12:53.846485289Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"6fe705dc-d7bb-4e42-85eb-6d520326ce3a","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":185,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.33024309Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52353"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.330400873Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.330445604Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.330464365Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.330485465Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.330492695Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.330501866Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.330508255Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.330518026Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"39cfbcbf-bc00-48f9-b1bd-2ab94ddc7242","remote_addr":"[fd00:1122:3344:101::b]:52353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":240,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a09fdb4-7bdd-4499-b8c4-37db9a5bc45c\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:13:15 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"6a09fdb4-7bdd-4499-b8c4-37db9a5bc45c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.413788246Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.717587392Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:38617"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.717652704Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"78434270-52ba-41dc-93c3-cc0191da5e37","remote_addr":"[fd00:1122:3344:101::b]:38617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.718443909Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.71846506Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.718852538Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:15.719037791Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"30567f7c-7986-40ee-8926-17929de9290a","remote_addr":"[fd00:1122:3344:101::b]:38617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1248,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:21.639211846Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64166"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:21.639266107Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"9b8f87e8-667e-47d7-9af2-f165d3c2dcb1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64166","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:23.84696956Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"9f9a270c-4317-47d8-8480-7919404d411f","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.350415789Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:59591"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.350567312Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.350611723Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.350635403Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.350640893Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.350646563Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.350653244Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.350660614Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.350685184Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c3908126-0464-47f0-8016-9528c90c0411","remote_addr":"[fd00:1122:3344:101::b]:59591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":238,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e9655029-28c4-463d-83f7-8ed4ffa71bf8\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:13:41 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"e9655029-28c4-463d-83f7-8ed4ffa71bf8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.435509195Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.607930713Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41544"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.607996764Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"3cf3eda8-46c5-41e7-b6e7-e5695e610cfe","remote_addr":"[fd00:1122:3344:101::b]:41544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.608137547Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.608157767Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.608378391Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:41.608529605Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"12f2577c-3581-4d78-83a5-01da080a97fe","remote_addr":"[fd00:1122:3344:101::b]:41544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":416,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:51.672715884Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60711"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:51.672768905Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"57425991-1ae0-421a-8123-1acbe9cd0bd0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60711","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:13:53.846551092Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"7b62d185-2fcb-4308-ab36-b1f524618b6f","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":270,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:21.704333844Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49133"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:21.704392585Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"745ecc81-99c5-49b4-8240-896147b6864b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49133","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":37,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:23.847765196Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"eb017b1c-99ca-4b0b-bb48-ed42db36acd1","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":788,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.750372024Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37682"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.750523097Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.750566968Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.750586849Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.750594549Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.750604399Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.750613039Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.750619579Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.75064886Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"55acceef-e6fd-4b86-b71a-a527affe2e9c","remote_addr":"[fd00:1122:3344:101::b]:37682","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":252,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2884537b-6937-4d8a-b69c-724736f9e0f4\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:14:33 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"2884537b-6937-4d8a-b69c-724736f9e0f4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.803916742Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.928934664Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33382"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.929009646Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"dd6cf21f-458b-4ecc-9f8c-3e50f7cc9c4b","remote_addr":"[fd00:1122:3344:101::b]:33382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":28,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.92921177Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.92922834Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.929635318Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:33.929822472Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"503a53e4-cd77-44d4-9680-27c567050f2b","remote_addr":"[fd00:1122:3344:101::b]:33382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":643,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:51.736322523Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56445"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:51.736376334Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"471d6912-71af-4569-9b4b-4d413f2680af","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56445","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:14:53.846600873Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"a21fac27-af64-4342-b6b0-82e54062bd23","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":179,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:21.769320641Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34427"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:21.769376723Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"796f8c45-2801-46f3-8e44-be0a35a7bc00","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34427","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":28,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:23.294500818Z","hostname":"sock","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:299"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:23.294563109Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:23.333941994Z","hostname":"sock","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.999997s","next_cleanup":"Instant { tv_sec: 2289, tv_nsec: 772430782 }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:23.855597344Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"f2c672c0-eb4f-4fe0-8b02-19b8aaf4ad9b","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8927,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:29.985821346Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56104"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:29.98603011Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:29.986076711Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:29.986093772Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:29.986104292Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:29.986120872Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:29.986131672Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:29.986142623Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:29.986166253Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ea15ce36-0b3d-4e65-bff1-a75469890891","remote_addr":"[fd00:1122:3344:101::b]:56104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":281,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9f658ad5-42fa-469e-b48d-b07f39e82aac\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:15:29 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"9f658ad5-42fa-469e-b48d-b07f39e82aac\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:30.059937174Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:30.188295503Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:36322"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:30.188895985Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"edc207f0-4967-46b5-ab1b-ce3227a40a28","remote_addr":"[fd00:1122:3344:101::b]:36322","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":553,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:30.189086269Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:30.189113549Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:30.189455166Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:30.18965437Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"674e837d-880f-4f0f-8e66-e9c23f60f432","remote_addr":"[fd00:1122:3344:101::b]:36322","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":608,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:51.800734519Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43473"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:51.800817161Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"e9d07999-c1e9-4570-b329-f4717d9242c3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43473","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":51,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:15:53.847165357Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"0eee740e-4b2f-468c-b55e-968df8468626","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":954,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:08.415625234Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:21.832324679Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:21.832393331Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"f336d330-1fc0-4e6f-ba4d-ed13475e4a1c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53020","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":44,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:23.846957493Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"1db11127-c951-4b40-a432-e256cc7e8731","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:29.298816992Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:51411"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:29.303544466Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:228","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-15T00:16:28.903818Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: Some([Vpc(Vni(16654092))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-9c6380e3d07\", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, read_only_parent: Some(Volume { block_size: 512, id: ad72fa73-db1d-4bda-86fd-e923832de49e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a89791d0-9b8a-48e3-b873-25fdd54a89e0, key: Some(\"CZUh9G+urVG1RRyq9KaZu+1vHje9xKhTjAM/2HEu5Bo=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, key: Some(\"VyAhAqjFfYr4vKeuRVRSMV66ERIy+JKPSvyW9BigcBo=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB+DgJPWE9YAAAOAk9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAfg4CT1hPWAAADgJPWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5NGZjY2I0MS0xNmEwLTRkZjAtYmNlNi1hMTdiMTRkNGJlMDUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRTFsZU1nNTdmUWlRZGgzRi81Qm14T0h3NUhPWXIzeGlVSVVuVHhkMGc2NyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:29.303623587Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:262","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:29.303897313Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance.rs:706","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: Some([Vpc(Vni(16654092))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-9c6380e3d07\", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, read_only_parent: Some(Volume { block_size: 512, id: ad72fa73-db1d-4bda-86fd-e923832de49e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a89791d0-9b8a-48e3-b873-25fdd54a89e0, key: Some(\"CZUh9G+urVG1RRyq9KaZu+1vHje9xKhTjAM/2HEu5Bo=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, key: Some(\"VyAhAqjFfYr4vKeuRVRSMV66ERIy+JKPSvyW9BigcBo=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB+DgJPWE9YAAAOAk9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAfg4CT1hPWAAADgJPWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5NGZjY2I0MS0xNmEwLTRkZjAtYmNlNi1hMTdiMTRkNGJlMDUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRTFsZU1nNTdmUWlRZGgzRi81Qm14T0h3NUhPWXIzeGlVSVVuVHhkMGc2NyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-15T00:16:28.903818Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:29.304652498Z","hostname":"sock","pid":1671,"uri":"/instances/94fccb41-16a0-4df0-bce6-a17b14d4be05","method":"PUT","req_id":"3caa88e2-50d7-45e8-94a1-68b6962215de","remote_addr":"[fd00:1122:3344:101::a]:51411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":5754,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:29.419000167Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:50453"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:29.441680339Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", _ip: 172.30.0.6, mac: MacAddr6([168, 64, 37, 254, 110, 222]), slot: 1, vni: Vni { inner: 16654092 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:29.441826732Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:29.477148166Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:29.499508972Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:31.567963519Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:31.568156493Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e\") failed: Property not found. retry in 60.83158ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:31.761424146Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e\") failed: Property not found. retry in 148.373225ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:31.8343777Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e\") failed: Property not found. retry in 111.975713ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:32.005094094Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e\") failed: Property not found. retry in 525.554745ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:32.130333291Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e\") failed: Property not found. retry in 1.092930954s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:32.666625052Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e\") failed: Property not found. retry in 644.661587ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:33.771080841Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e\") failed: Property not found. retry in 658.38205ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:34.426067308Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e\") failed: Property not found. retry in 975.423708ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:35.093948973Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e\") failed: Property not found. retry in 1.321263064s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:36.078608864Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.427410463Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:1098"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.435473574Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:1107"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.469719537Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:1121"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB+DgJPWE9YAAAOAk9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAfg4CT1hPWAAADgJPWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5NGZjY2I0MS0xNmEwLTRkZjAtYmNlNi1hMTdiMTRkNGJlMDUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRTFsZU1nNTdmUWlRZGgzRi81Qm14T0h3NUhPWXIzeGlVSVVuVHhkMGc2NyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-9c6380e3d07\", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, read_only_parent: Some(Volume { block_size: 512, id: ad72fa73-db1d-4bda-86fd-e923832de49e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a89791d0-9b8a-48e3-b873-25fdd54a89e0, key: Some(\"CZUh9G+urVG1RRyq9KaZu+1vHje9xKhTjAM/2HEu5Bo=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, key: Some(\"VyAhAqjFfYr4vKeuRVRSMV66ERIy+JKPSvyW9BigcBo=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(1) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 94fccb41-16a0-4df0-bce6-a17b14d4be05, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.470219416Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:453"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.621119625Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:455"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-02-15T00:16:37.622201717Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.622223247Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.622244767Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-15T00:16:37.622201717Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }","propolis_id":"d13d05bd-eed0-4965-8d7c-dde7b491d43e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.622252867Z","hostname":"sock","pid":1671,"uri":"/instances/94fccb41-16a0-4df0-bce6-a17b14d4be05/state","method":"PUT","req_id":"e7bec46f-2aa9-4166-80c8-bbc7f38812d6","remote_addr":"[fd00:1122:3344:101::a]:50453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8203163,"response_code":"200"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.62238777Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }, propolis_id: d13d05bd-eed0-4965-8d7c-dde7b491d43e, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-15T00:16:37.622201717Z } }","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-02-15T00:16:37.675623191Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.675640022Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.675659142Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-15T00:16:37.675623191Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }","propolis_id":"d13d05bd-eed0-4965-8d7c-dde7b491d43e"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:37.675669103Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }, propolis_id: d13d05bd-eed0-4965-8d7c-dde7b491d43e, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-15T00:16:37.675623191Z } }","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.577420702Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52794"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.577575975Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name(\"external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name(\"external-dns-1999ec12-789c-40c8-83be-a68bad0fad69\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name(\"nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name(\"nexus-052b5be3-98fd-4823-9614-96d5055e333c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name(\"nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.577625636Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.577647626Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.577676957Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.577683807Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.577690667Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.577697298Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.577704838Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2b8143f3-306a-4f05-8eaf-17d0c6fdcbc4","remote_addr":"[fd00:1122:3344:101::b]:52794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":256,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d91b6119-1ea2-46e7-9d0c-ec097b7bcb36\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 00:16:45 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"d91b6119-1ea2-46e7-9d0c-ec097b7bcb36\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.630547421Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.725347881Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41255"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.725486544Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"886b30b3-d4d4-408d-ad08-fcbeb9b146fc","remote_addr":"[fd00:1122:3344:101::b]:41255","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":67,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.726198738Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.726228599Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.726718318Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:45.726921802Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"45dec3c3-1111-400f-a7f9-8b8b2adf4836","remote_addr":"[fd00:1122:3344:101::b]:41255","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1223,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:51.866544862Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:51.867140644Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"c3da4b6a-e580-473d-a2e4-c8c08bbf7e96","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38346","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":569,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:53.847079996Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a","method":"GET","req_id":"6778ad12-d6a1-441d-a776-bf1b251a0fa7","remote_addr":"[fd00:1122:3344:101::d]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":280,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.573809964Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40058"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.574253993Z","hostname":"sock","pid":1671,"uri":"/instances/94fccb41-16a0-4df0-bce6-a17b14d4be05/state","method":"PUT","req_id":"0e3dee90-f096-4434-8926-6f8c5c0c5e83","remote_addr":"[fd00:1122:3344:101::a]:40058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":400,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-02-15T00:16:54.574309394Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.574327015Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.574340035Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-15T00:16:54.574309394Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }","propolis_id":"d13d05bd-eed0-4965-8d7c-dde7b491d43e"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.574354375Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }, propolis_id: d13d05bd-eed0-4965-8d7c-dde7b491d43e, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-15T00:16:54.574309394Z } }","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopped), migration_status: NoMigration, time: 2024-02-15T00:16:54.593941555Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.593957556Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.593974326Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-15T00:16:54.593941555Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }","propolis_id":"d13d05bd-eed0-4965-8d7c-dde7b491d43e"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.593984336Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }, propolis_id: d13d05bd-eed0-4965-8d7c-dde7b491d43e, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-15T00:16:54.593941555Z } }","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-02-15T00:16:54.756784642Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.756813553Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.756834513Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-02-15T00:16:54.756784642Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-15T00:16:54.756784642Z }","propolis_id":"d13d05bd-eed0-4965-8d7c-dde7b491d43e"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.756844403Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:374","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.756850883Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.756996726Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:54.757047677Z","hostname":"sock","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:436","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\", \"/pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/debug\", \"/pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/debug\", \"/pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/debug\", \"/pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/debug\", \"/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/debug\", \"/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\"] }","zone_name":"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:55.34156858Z","hostname":"sock","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1229","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e\", bundle_id: 2c3db916-2f68-46c5-aef6-05abe9267bc0 }, time_created: 2024-02-15T00:16:54.757895464Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:55.341688393Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:547"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:56.912352495Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:56.938605009Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-15T00:16:54.756784642Z }, propolis_id: d13d05bd-eed0-4965-8d7c-dde7b491d43e, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-02-15T00:16:54.756784642Z } }","instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:57.012653285Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:40009"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T00:16:57.013556743Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:57.013589624Z","hostname":"sock","pid":1671,"uri":"/v2p/82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba","method":"DELETE","req_id":"e4e03acd-816e-4067-b072-7625488be599","remote_addr":"[fd00:1122:3344:101::b]:40009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":835,"response_code":"204"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T00:16:57.108407704Z","hostname":"sock","pid":1671,"instance_id":"94fccb41-16a0-4df0-bce6-a17b14d4be05","component":"InstanceManager","file":"sled-agent/src/instance.rs:490"}