| 1 | | [ Feb 15 00:05:21 Enabled. ] |
| 2 | | [ Feb 15 00:05:21 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
| 3 | | [ Feb 15 00:05:21 Method "start" exited with status 0. ] |
| 4 | | [ Feb 15 00:05:21 Rereading configuration. ] |
| 5 | | [ Feb 15 00:05:21 No 'refresh' method defined. Treating as :true. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2024-02-15T00:05:21.616Z | INFO | SledAgent (BootstrapAgentStartup): finding links ["net0", "net1"]
file = sled-agent/src/bootstrap/pre_server.rs:247
|
| 8 | 2024-02-15T00:05:21.670Z | INFO | SledAgent (BootstrapAgentStartup): Starting mg-ddm service
addr-objs = [AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]
file = sled-agent/src/bootstrap/pre_server.rs:254
|
| 9 | 2024-02-15T00:05:21.670Z | INFO | SledAgent (BootstrapAgentStartup): Importing mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:42
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
| 10 | 2024-02-15T00:05:21.783Z | INFO | SledAgent (BootstrapAgentStartup): Setting mg-ddm interfaces
file = sled-agent/src/bootstrap/maghemite.rs:50
interfaces = ("net0/ll" "net1/ll")
|
| 11 | 2024-02-15T00:05:21.792Z | INFO | SledAgent (BootstrapAgentStartup): Enabling mg-ddm service
file = sled-agent/src/bootstrap/maghemite.rs:56
|
| 12 | 2024-02-15T00:05:21.799Z | INFO | SledAgent (BootstrapAgentStartup): Ensuring zfs key directory exists
file = sled-agent/src/bootstrap/pre_server.rs:264
path = /var/run/oxide/
|
| 13 | 2024-02-15T00:05:22.405Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:91
prefix = Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }
|
| 14 | 2024-02-15T00:05:22.543Z | INFO | SledAgent: Starting KeyManager
file = sled-agent/src/long_running_tasks.rs:117
|
| 15 | 2024-02-15T00:05:22.543Z | INFO | SledAgent: Starting StorageManager
file = sled-agent/src/long_running_tasks.rs:129
|
| 16 | 2024-02-15T00:05:22.543Z | INFO | SledAgent: Starting StorageMonitor
file = sled-agent/src/long_running_tasks.rs:141
|
| 17 | 2024-02-15T00:05:22.543Z | INFO | SledAgent: Starting HardwareManager
file = sled-agent/src/long_running_tasks.rs:162
sled_mode = Scrimlet { asic: SoftNpuZone }
|
| 18 | 2024-02-15T00:05:22.543Z | INFO | SledAgent (HardwareManager): Creating HardwareManager
file = sled-hardware/src/illumos/mod.rs:609
|
| 19 | 2024-02-15T00:05:22.543Z | INFO | SledAgent (DumpSetup-archival): DumpSetup poll loop started.
file = sled-agent/src/dump_setup.rs:268
|
| 20 | 2024-02-15T00:05:22.544Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:776
|
| 21 | 2024-02-15T00:05:22.547Z | INFO | SledAgent (HardwareManager): Generated i86pc baseboard Pc { identifier: "sock", model: "i86pc" }
file = sled-hardware/src/illumos/mod.rs:535
|
| 22 | 2024-02-15T00:05:22.547Z | WARN | SledAgent (HardwareManager): Device is not a Gimlet (i86pc), proceeding with null hardware view
file = sled-hardware/src/illumos/mod.rs:658
|
| 23 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Starting HardwareMonitor
file = sled-agent/src/long_running_tasks.rs:176
|
| 24 | 2024-02-15T00:05:22.547Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
file = sled-hardware/src/illumos/mod.rs:716
|
| 25 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b
file = sled-agent/src/long_running_tasks.rs:232
|
| 26 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b
file = sled-agent/src/long_running_tasks.rs:232
|
| 27 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
file = sled-agent/src/long_running_tasks.rs:232
|
| 28 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
file = sled-agent/src/long_running_tasks.rs:232
|
| 29 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
file = sled-agent/src/long_running_tasks.rs:232
|
| 30 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03
file = sled-agent/src/long_running_tasks.rs:232
|
| 31 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03
file = sled-agent/src/long_running_tasks.rs:232
|
| 32 | 2024-02-15T00:05:22.547Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
file = sled-agent/src/hardware_monitor.rs:241
underlay_network_info = None
|
| 33 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930
file = sled-agent/src/long_running_tasks.rs:232
|
| 34 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60
file = sled-agent/src/long_running_tasks.rs:232
|
| 35 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxp_31bd71cd-4736-4a12-a387-9b74b050396f
file = sled-agent/src/long_running_tasks.rs:232
|
| 36 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Upserting synthetic zpool to Storage Manager: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7
file = sled-agent/src/long_running_tasks.rs:232
|
| 37 | 2024-02-15T00:05:22.547Z | INFO | SledAgent: Waiting for boot disk
file = sled-agent/src/long_running_tasks.rs:88
|
| 38 | 2024-02-15T00:05:22.547Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))
file = sled-storage/src/manager.rs:346
|
| 39 | 2024-02-15T00:05:22.726Z | INFO | SledAgent (StorageManager): Channel capacity decreased
current = 245
file = sled-storage/src/manager.rs:326
previous = 256
|
| 40 | 2024-02-15T00:05:22.726Z | INFO | SledAgent: Found boot disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
file = sled-agent/src/long_running_tasks.rs:90
|
| 41 | 2024-02-15T00:05:22.726Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))
file = sled-storage/src/manager.rs:346
|
| 42 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }))
file = sled-storage/src/manager.rs:346
|
| 43 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))
file = sled-storage/src/manager.rs:346
|
| 44 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))
file = sled-storage/src/manager.rs:346
|
| 45 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))
file = sled-storage/src/manager.rs:346
|
| 46 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))
file = sled-storage/src/manager.rs:346
|
| 47 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", model: "synthetic-model" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }))
file = sled-storage/src/manager.rs:346
|
| 48 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", model: "synthetic-model" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }))
file = sled-storage/src/manager.rs:346
|
| 49 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", model: "synthetic-model" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }))
file = sled-storage/src/manager.rs:346
|
| 50 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", model: "synthetic-model" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }))
file = sled-storage/src/manager.rs:346
|
| 51 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received DisksChanged({})
file = sled-storage/src/manager.rs:346
|
| 52 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Channel capacity increased
current = 255
file = sled-storage/src/manager.rs:336
previous = 245
|
| 53 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 54 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (StorageMonitor): Received storage manager update
file = sled-agent/src/storage_monitor.rs:111
resources = StorageResources { disks: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: "oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b", size: 20937965568, allocated: 503808, free: 20937461760, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: "oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b", size: 20937965568, allocated: 491520, free: 20937474048, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" } })} }
|
| 55 | 2024-02-15T00:05:22.893Z | INFO | SledAgent: Starting Bootstore
file = sled-agent/src/long_running_tasks.rs:200
|
| 56 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (bootstore): No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:187
peer_id = pc-sock-i86pc
|
| 57 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (bootstore): No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json
file = common/src/ledger.rs:187
peer_id = pc-sock-i86pc
|
| 58 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (bootstore): No ledger found. Loading Fsm::State as Uninitialized
file = bootstore/src/schemes/v0/storage.rs:83
peer_id = pc-sock-i86pc
|
| 59 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (bootstore): No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json
file = common/src/ledger.rs:187
peer_id = pc-sock-i86pc
|
| 60 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (bootstore): No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json
file = common/src/ledger.rs:187
peer_id = pc-sock-i86pc
|
| 61 | 2024-02-15T00:05:22.893Z | INFO | SledAgent (bootstore): No ledger found for network config
file = bootstore/src/schemes/v0/storage.rs:146
peer_id = pc-sock-i86pc
|
| 62 | 2024-02-15T00:05:22.893Z | INFO | SledAgent: Starting Bootstore DDMD poller
file = sled-agent/src/long_running_tasks.rs:205
|
| 63 | 2024-02-15T00:05:22.893Z | INFO | SledAgent: Starting ZoneBundler related tasks
file = sled-agent/src/long_running_tasks.rs:220
|
| 64 | 2024-02-15T00:05:22.908Z | INFO | SledAgent (ServiceManager): Creating ServiceManager
file = sled-agent/src/services.rs:697
|
| 65 | 2024-02-15T00:05:22.908Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:288
next_cleanup = Instant { tv_sec: 1689, tv_nsec: 731539276 }
time_to_next_cleanup = 599.999928038s
|
| 66 | 2024-02-15T00:05:22.908Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3345
|
| 67 | 2024-02-15T00:05:22.908Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 68 | 2024-02-15T00:05:22.908Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json
file = common/src/ledger.rs:187
|
| 69 | 2024-02-15T00:05:22.908Z | INFO | SledAgent (BootstrapAgentStartup): No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json
file = common/src/ledger.rs:187
|
| 70 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (dropshot (BootstrapAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
|
| 71 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (SledAgentSprocketsServer): Started listening
file = sled-agent/src/bootstrap/sprockets_server.rs:44
local_addr = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 72 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (RSS): Injecting RSS configuration: RackInitializeRequest {
trust_quorum_peers: None,
bootstrap_discovery: OnlyOurs,
ntp_servers: [
"0.pool.ntp.org",
],
dns_servers: [
1.1.1.1,
9.9.9.9,
],
internal_services_ip_pool_ranges: [
V4(
Ipv4Range {
first: 10.151.2.164,
last: 10.151.2.173,
},
),
],
external_dns_ips: [
10.151.2.164,
10.151.2.165,
],
external_dns_zone_name: "oxide.test",
external_certificates: "<redacted>",
recovery_silo: RecoverySiloConfig {
silo_name: Name(
"recovery",
),
user_name: UserId(
"recovery",
),
user_password_hash: NewPasswordHash(
PasswordHashString {
string: "$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY",
encoding: B64,
},
),
},
rack_network_config: RackNetworkConfigV1 {
rack_subnet: Ipv6Network {
addr: fd00:1122:3344:100::,
prefix: 56,
},
infra_ip_first: 192.168.1.30,
infra_ip_last: 192.168.1.30,
ports: [
PortConfigV1 {
routes: [
RouteConfig {
destination: V4(
Ipv4Network {
addr: 0.0.0.0,
prefix: 0,
},
),
nexthop: 10.151.2.100,
},
],
addresses: [
V4(
Ipv4Network {
addr: 10.151.2.174,
prefix: 24,
},
),
],
switch: Switch0,
port: "qsfp0",
uplink_port_speed: Speed40G,
uplink_port_fec: None,
bgp_peers: [],
autoneg: false,
},
],
bgp: [],
},
}
file = sled-agent/src/rack_setup/service.rs:800
|
| 73 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 74 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 75 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (RSS): No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker
file = common/src/ledger.rs:187
|
| 76 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (RSS): No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker
file = common/src/ledger.rs:187
|
| 77 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
file = sled-agent/src/rack_setup/service.rs:860
|
| 78 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 79 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (RSS): No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json
file = common/src/ledger.rs:187
|
| 80 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (RSS): No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json
file = common/src/ledger.rs:187
|
| 81 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (RSS): Creating new allocation plan
file = sled-agent/src/rack_setup/service.rs:901
|
| 82 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (RSS): Serializing plan
file = sled-agent/src/rack_setup/plan/sled.rs:118
|
| 83 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:a02a::1
file = sled-agent/src/rack_setup/plan/sled.rs:91
|
| 84 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 85 | 2024-02-15T00:05:22.912Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:198
|
| 86 | 2024-02-15T00:05:22.913Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp
file = common/src/ledger.rs:198
|
| 87 | 2024-02-15T00:05:22.913Z | INFO | SledAgent (RSS): Sled plan written to storage
file = sled-agent/src/rack_setup/plan/sled.rs:138
|
| 88 | 2024-02-15T00:05:22.913Z | INFO | SledAgent (RSS): Writing Rack Network Configuration to bootstore
file = sled-agent/src/rack_setup/service.rs:934
|
| 89 | 2024-02-15T00:05:22.913Z | INFO | SledAgent (bootstore): Attempting to update network config with generation: 1, current_generation: 0
file = bootstore/src/schemes/v0/peer.rs:564
peer_id = pc-sock-i86pc
|
| 90 | 2024-02-15T00:05:22.913Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:198
peer_id = pc-sock-i86pc
|
| 91 | 2024-02-15T00:05:22.913Z | INFO | SledAgent (bootstore): Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp
file = common/src/ledger.rs:198
peer_id = pc-sock-i86pc
|
| 92 | 2024-02-15T00:05:22.913Z | INFO | SledAgent (bootstore): Broadcasting network config with generation 1
file = bootstore/src/schemes/v0/peer.rs:648
peer_id = pc-sock-i86pc
|
| 93 | 2024-02-15T00:05:22.913Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
file = sled-agent/src/bootstrap/rss_handle.rs:217
request = StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, rack_id: fdf13508-1e61-48db-a883-3beaba642d42, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 94 | 2024-02-15T00:05:22.914Z | INFO | SledAgent (SledAgentSprocketsServer): Accepted connection
file = sled-agent/src/bootstrap/sprockets_server.rs:66
remote_addr = [fdb0:18c0:4d0d:a02a::1]:55221
|
| 95 | 2024-02-15T00:05:22.914Z | INFO | SledAgent (SledAgentMain): Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, rack_id: fdf13508-1e61-48db-a883-3beaba642d42, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }
file = sled-agent/src/bootstrap/server.rs:364
|
| 96 | 2024-02-15T00:05:22.914Z | INFO | SledAgent (SledAgentMain): KeyManager: using hardcoded secret retriever
file = sled-agent/src/bootstrap/server.rs:379
|
| 97 | 2024-02-15T00:05:22.914Z | INFO | SledAgent: setting up sled agent server
file = sled-agent/src/server.rs:47
|
| 98 | 2024-02-15T00:05:22.914Z | INFO | SledAgent (StorageManager): Received KeyManagerReady
file = sled-storage/src/manager.rs:346
|
| 99 | 2024-02-15T00:05:22.914Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 100 | 2024-02-15T00:05:22.914Z | INFO | SledAgent (StorageManager): Attempting to add queued disks
file = sled-storage/src/manager.rs:394
num_disks = 9
|
| 101 | 2024-02-15T00:05:22.916Z | INFO | SledAgent (ServiceManager): Enabling switch zone (new)
file = sled-agent/src/services.rs:3573
|
| 102 | 2024-02-15T00:05:22.922Z | INFO | SledAgent (StorageManager): Loading latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
file = sled-storage/src/dataset.rs:327
|
| 103 | 2024-02-15T00:05:22.922Z | INFO | SledAgent (StorageManager): Loaded latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:329
|
| 104 | 2024-02-15T00:05:22.922Z | INFO | SledAgent (StorageManager): Retrieving key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:333
|
| 105 | 2024-02-15T00:05:22.923Z | INFO | SledAgent (StorageManager): Got key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:335
|
| 106 | 2024-02-15T00:05:22.923Z | INFO | SledAgent (StorageManager): Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:37
|
| 107 | 2024-02-15T00:05:22.923Z | INFO | SledAgent (StorageManager): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:347
|
| 108 | 2024-02-15T00:05:22.955Z | INFO | SledAgent (StorageManager): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:56
|
| 109 | 2024-02-15T00:05:22.963Z | INFO | SledAgent (StorageManager): Automatically destroying dataset: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone
file = sled-storage/src/dataset.rs:389
|
| 110 | 2024-02-15T00:05:22.993Z | INFO | SledAgent: SledAgent::new(..) starting
file = sled-agent/src/sled_agent.rs:342
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 111 | 2024-02-15T00:05:22.993Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:91
prefix = Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }
|
| 112 | 2024-02-15T00:05:23.068Z | INFO | SledAgent (StorageManager): Looking for unencrypted datasets in oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
file = sled-storage/src/dataset.rs:474
|
| 113 | 2024-02-15T00:05:23.091Z | INFO | SledAgent (StorageManager): Loading latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7",\n model: "synthetic-model",\n}
file = sled-storage/src/dataset.rs:327
|
| 114 | 2024-02-15T00:05:23.091Z | INFO | SledAgent (StorageManager): Loaded latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:329
|
| 115 | 2024-02-15T00:05:23.091Z | INFO | SledAgent (StorageManager): Retrieving key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:333
|
| 116 | 2024-02-15T00:05:23.091Z | INFO | SledAgent (StorageManager): Got key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:335
|
| 117 | 2024-02-15T00:05:23.091Z | INFO | SledAgent (StorageManager): Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:37
|
| 118 | 2024-02-15T00:05:23.091Z | INFO | SledAgent (StorageManager): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:347
|
| 119 | 2024-02-15T00:05:23.117Z | INFO | SledAgent (StorageManager): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:56
|
| 120 | 2024-02-15T00:05:23.124Z | INFO | SledAgent (StorageManager): Automatically destroying dataset: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone
file = sled-storage/src/dataset.rs:389
|
| 121 | 2024-02-15T00:05:23.228Z | INFO | SledAgent (StorageManager): Looking for unencrypted datasets in oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7
file = sled-storage/src/dataset.rs:474
|
| 122 | 2024-02-15T00:05:23.249Z | INFO | SledAgent (StorageManager): Loading latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
file = sled-storage/src/dataset.rs:327
|
| 123 | 2024-02-15T00:05:23.250Z | INFO | SledAgent (StorageManager): Loaded latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:329
|
| 124 | 2024-02-15T00:05:23.250Z | INFO | SledAgent (StorageManager): Retrieving key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:333
|
| 125 | 2024-02-15T00:05:23.250Z | INFO | SledAgent (StorageManager): Got key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:335
|
| 126 | 2024-02-15T00:05:23.250Z | INFO | SledAgent (StorageManager): Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:37
|
| 127 | 2024-02-15T00:05:23.250Z | INFO | SledAgent (StorageManager): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:347
|
| 128 | 2024-02-15T00:05:23.276Z | INFO | SledAgent (StorageManager): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:56
|
| 129 | 2024-02-15T00:05:23.283Z | INFO | SledAgent (StorageManager): Automatically destroying dataset: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone
file = sled-storage/src/dataset.rs:389
|
| 130 | 2024-02-15T00:05:23.387Z | INFO | SledAgent (StorageManager): Looking for unencrypted datasets in oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03
file = sled-storage/src/dataset.rs:474
|
| 131 | 2024-02-15T00:05:23.410Z | INFO | SledAgent (StorageManager): Loading latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
file = sled-storage/src/dataset.rs:327
|
| 132 | 2024-02-15T00:05:23.410Z | INFO | SledAgent (StorageManager): Loaded latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:329
|
| 133 | 2024-02-15T00:05:23.410Z | INFO | SledAgent (StorageManager): Retrieving key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:333
|
| 134 | 2024-02-15T00:05:23.410Z | INFO | SledAgent (StorageManager): Got key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:335
|
| 135 | 2024-02-15T00:05:23.410Z | INFO | SledAgent (StorageManager): Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:37
|
| 136 | 2024-02-15T00:05:23.410Z | INFO | SledAgent (StorageManager): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:347
|
| 137 | 2024-02-15T00:05:23.438Z | INFO | SledAgent (StorageManager): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:56
|
| 138 | 2024-02-15T00:05:23.446Z | INFO | SledAgent (StorageManager): Automatically destroying dataset: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone
file = sled-storage/src/dataset.rs:389
|
| 139 | 2024-02-15T00:05:23.547Z | INFO | SledAgent (StorageManager): Looking for unencrypted datasets in oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03
file = sled-storage/src/dataset.rs:474
|
| 140 | 2024-02-15T00:05:23.569Z | INFO | SledAgent (StorageManager): Loading latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f",\n model: "synthetic-model",\n}
file = sled-storage/src/dataset.rs:327
|
| 141 | 2024-02-15T00:05:23.569Z | INFO | SledAgent (StorageManager): Loaded latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:329
|
| 142 | 2024-02-15T00:05:23.569Z | INFO | SledAgent (StorageManager): Retrieving key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:333
|
| 143 | 2024-02-15T00:05:23.569Z | INFO | SledAgent (StorageManager): Got key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:335
|
| 144 | 2024-02-15T00:05:23.569Z | INFO | SledAgent (StorageManager): Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:37
|
| 145 | 2024-02-15T00:05:23.569Z | INFO | SledAgent (StorageManager): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:347
|
| 146 | 2024-02-15T00:05:23.597Z | INFO | SledAgent (StorageManager): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:56
|
| 147 | 2024-02-15T00:05:23.604Z | INFO | SledAgent (StorageManager): Automatically destroying dataset: oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone
file = sled-storage/src/dataset.rs:389
|
| 148 | 2024-02-15T00:05:23.709Z | INFO | SledAgent (StorageManager): Looking for unencrypted datasets in oxp_31bd71cd-4736-4a12-a387-9b74b050396f
file = sled-storage/src/dataset.rs:474
|
| 149 | 2024-02-15T00:05:23.731Z | INFO | SledAgent (StorageManager): Loading latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",\n model: "synthetic-model",\n}
file = sled-storage/src/dataset.rs:327
|
| 150 | 2024-02-15T00:05:23.731Z | INFO | SledAgent (StorageManager): Loaded latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:329
|
| 151 | 2024-02-15T00:05:23.731Z | INFO | SledAgent (StorageManager): Retrieving key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:333
|
| 152 | 2024-02-15T00:05:23.731Z | INFO | SledAgent (StorageManager): Got key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:335
|
| 153 | 2024-02-15T00:05:23.731Z | INFO | SledAgent (StorageManager): Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:37
|
| 154 | 2024-02-15T00:05:23.731Z | INFO | SledAgent (StorageManager): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:347
|
| 155 | 2024-02-15T00:05:23.759Z | INFO | SledAgent (StorageManager): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:56
|
| 156 | 2024-02-15T00:05:23.767Z | INFO | SledAgent (StorageManager): Automatically destroying dataset: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone
file = sled-storage/src/dataset.rs:389
|
| 157 | 2024-02-15T00:05:23.870Z | INFO | SledAgent (StorageManager): Looking for unencrypted datasets in oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
file = sled-storage/src/dataset.rs:474
|
| 158 | 2024-02-15T00:05:23.892Z | INFO | SledAgent (StorageManager): Loading latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60",\n model: "synthetic-model",\n}
file = sled-storage/src/dataset.rs:327
|
| 159 | 2024-02-15T00:05:23.892Z | INFO | SledAgent (StorageManager): Loaded latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:329
|
| 160 | 2024-02-15T00:05:23.892Z | INFO | SledAgent (StorageManager): Retrieving key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:333
|
| 161 | 2024-02-15T00:05:23.892Z | INFO | SledAgent (StorageManager): Got key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:335
|
| 162 | 2024-02-15T00:05:23.892Z | INFO | SledAgent (StorageManager): Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:37
|
| 163 | 2024-02-15T00:05:23.892Z | INFO | SledAgent (StorageManager): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:347
|
| 164 | 2024-02-15T00:05:23.919Z | INFO | SledAgent (StorageManager): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:56
|
| 165 | 2024-02-15T00:05:23.927Z | INFO | SledAgent (StorageManager): Automatically destroying dataset: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone
file = sled-storage/src/dataset.rs:389
|
| 166 | 2024-02-15T00:05:24.032Z | INFO | SledAgent (StorageManager): Looking for unencrypted datasets in oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60
file = sled-storage/src/dataset.rs:474
|
| 167 | 2024-02-15T00:05:24.054Z | INFO | SledAgent (StorageManager): Loading latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930",\n model: "synthetic-model",\n}
file = sled-storage/src/dataset.rs:327
|
| 168 | 2024-02-15T00:05:24.054Z | INFO | SledAgent (StorageManager): Loaded latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:329
|
| 169 | 2024-02-15T00:05:24.054Z | INFO | SledAgent (StorageManager): Retrieving key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:333
|
| 170 | 2024-02-15T00:05:24.054Z | INFO | SledAgent (StorageManager): Got key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:335
|
| 171 | 2024-02-15T00:05:24.055Z | INFO | SledAgent (StorageManager): Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:37
|
| 172 | 2024-02-15T00:05:24.055Z | INFO | SledAgent (StorageManager): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:347
|
| 173 | 2024-02-15T00:05:24.083Z | INFO | SledAgent (StorageManager): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:56
|
| 174 | 2024-02-15T00:05:24.091Z | INFO | SledAgent (StorageManager): Automatically destroying dataset: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone
file = sled-storage/src/dataset.rs:389
|
| 175 | 2024-02-15T00:05:24.196Z | INFO | SledAgent (StorageManager): Looking for unencrypted datasets in oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930
file = sled-storage/src/dataset.rs:474
|
| 176 | 2024-02-15T00:05:24.218Z | INFO | SledAgent (StorageManager): Loading latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
file = sled-storage/src/dataset.rs:327
|
| 177 | 2024-02-15T00:05:24.219Z | INFO | SledAgent (StorageManager): Loaded latest secret
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:329
|
| 178 | 2024-02-15T00:05:24.219Z | INFO | SledAgent (StorageManager): Retrieving key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:333
|
| 179 | 2024-02-15T00:05:24.219Z | INFO | SledAgent (StorageManager): Got key
disk_id = DiskIdentity {\n vendor: "synthetic-vendor",\n serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",\n model: "synthetic-model",\n}
epoch = 0
file = sled-storage/src/dataset.rs:335
|
| 180 | 2024-02-15T00:05:24.219Z | INFO | SledAgent (StorageManager): Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:37
|
| 181 | 2024-02-15T00:05:24.219Z | INFO | SledAgent (StorageManager): Ensuring encrypted filesystem: crypt for epoch 0
file = sled-storage/src/dataset.rs:347
|
| 182 | 2024-02-15T00:05:24.247Z | INFO | SledAgent (StorageManager): Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key
file = sled-storage/src/keyfile.rs:56
|
| 183 | 2024-02-15T00:05:24.255Z | INFO | SledAgent (StorageManager): Automatically destroying dataset: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone
file = sled-storage/src/dataset.rs:389
|
| 184 | 2024-02-15T00:05:24.361Z | INFO | SledAgent (StorageManager): Looking for unencrypted datasets in oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
file = sled-storage/src/dataset.rs:474
|
| 185 | 2024-02-15T00:05:24.376Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 186 | 2024-02-15T00:05:24.376Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 187 | 2024-02-15T00:05:24.376Z | INFO | SledAgent: Requested swap device of size 64 GiB
file = sled-agent/src/sled_agent.rs:354
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 188 | 2024-02-15T00:05:24.379Z | INFO | SledAgent (StorageMonitor): Received storage manager update
file = sled-agent/src/storage_monitor.rs:111
resources = StorageResources { disks: {DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", model: "synthetic-model" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }), Pool { name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, info: ZpoolInfo { name: "oxp_31bd71cd-4736-4a12-a387-9b74b050396f", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", model: "synthetic-model" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }), Pool { name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, info: ZpoolInfo { name: "oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: "oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b", size: 20937965568, allocated: 503808, free: 20937461760, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: "oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b", size: 20937965568, allocated: 491520, free: 20937474048, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", model: "synthetic-model" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }), Pool { name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, info: ZpoolInfo { name: "oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930", size: 20937965568, allocated: 826368, free: 20937139200, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", model: "synthetic-model" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }), Pool { name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, info: ZpoolInfo { name: "oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60", size: 20937965568, allocated: 768000, free: 20937197568, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }), Pool { name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, info: ZpoolInfo { name: "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b", size: 20937965568, allocated: 780288, free: 20937185280, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03", size: 20937965568, allocated: 768000, free: 20937197568, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" } }), DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" } })} }
|
| 189 | 2024-02-15T00:05:24.399Z | INFO | SledAgent: attempting to create encrypted zvol: name="oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap", size_gb=64
file = sled-agent/src/swap_device.rs:153
|
| 190 | 2024-02-15T00:05:24.401Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:315
|
| 191 | 2024-02-15T00:05:24.419Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_switch
file = illumos-utils/src/zone.rs:354
|
| 192 | 2024-02-15T00:05:24.422Z | INFO | SledAgent: successfully created encrypted zvol: name="oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap", size_gb=64
file = sled-agent/src/swap_device.rs:249
|
| 193 | 2024-02-15T00:05:24.422Z | INFO | SledAgent: adding swap device: swapname="/dev/zvol/dsk/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap"
file = sled-agent/src/swap_device.rs:96
|
| 194 | 2024-02-15T00:05:24.427Z | INFO | SledAgent: Mounting backing filesystems
file = sled-agent/src/sled_agent.rs:369
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 195 | 2024-02-15T00:05:24.427Z | INFO | SledAgent (BackingFs): Processing fmd
file = sled-agent/src/backing_fs.rs:126
|
| 196 | 2024-02-15T00:05:24.427Z | INFO | SledAgent (BackingFs): Ensuring dataset oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd
file = sled-agent/src/backing_fs.rs:136
|
| 197 | 2024-02-15T00:05:24.490Z | INFO | SledAgent (BackingFs): Stopping service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:170
|
| 198 | 2024-02-15T00:05:24.709Z | INFO | SledAgent (BackingFs): Mounting oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd on /var/fm/fmd
file = sled-agent/src/backing_fs.rs:178
|
| 199 | 2024-02-15T00:05:24.721Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/rsrc
file = sled-agent/src/backing_fs.rs:186
|
| 200 | 2024-02-15T00:05:24.721Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/ckpt
file = sled-agent/src/backing_fs.rs:186
|
| 201 | 2024-02-15T00:05:24.721Z | INFO | SledAgent (BackingFs): Ensuring directory /var/fm/fmd/xprt
file = sled-agent/src/backing_fs.rs:186
|
| 202 | 2024-02-15T00:05:24.721Z | INFO | SledAgent (BackingFs): Starting service svc:/system/fmd:default
file = sled-agent/src/backing_fs.rs:192
|
| 203 | 2024-02-15T00:05:24.888Z | INFO | SledAgent: Ensuring contract reaper thread
file = illumos-utils/src/running_zone.rs:163
|
| 204 | 2024-02-15T00:05:25.426Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]' as data links for xde driver
file = illumos-utils/src/opte/illumos.rs:90
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 205 | 2024-02-15T00:05:25.426Z | INFO | SledAgent (StorageMonitor): Underlay Available
file = sled-agent/src/storage_monitor.rs:122
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 206 | 2024-02-15T00:05:25.426Z | INFO | SledAgent (InstanceManager): 50% of 102952026112 physical ram = 51476013056 bytes)
file = sled-agent/src/instance_manager.rs:178
|
| 207 | 2024-02-15T00:05:25.426Z | INFO | SledAgent (InstanceManager): Setting reservoir size to 49090 MiB bytes
file = sled-agent/src/instance_manager.rs:186
|
| 208 | 2024-02-15T00:05:25.441Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 209 | 2024-02-15T00:05:25.441Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 210 | 2024-02-15T00:05:25.441Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 211 | 2024-02-15T00:05:25.442Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 212 | 2024-02-15T00:05:25.442Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 213 | 2024-02-15T00:05:25.442Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 214 | 2024-02-15T00:05:25.442Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 215 | 2024-02-15T00:05:25.442Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 216 | 2024-02-15T00:05:25.442Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 217 | 2024-02-15T00:05:25.442Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 218 | 2024-02-15T00:05:25.443Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 219 | 2024-02-15T00:05:25.443Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 220 | 2024-02-15T00:05:25.443Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 221 | 2024-02-15T00:05:25.443Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:219
|
| 222 | 2024-02-15T00:05:25.443Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 223 | 2024-02-15T00:05:25.443Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 224 | 2024-02-15T00:05:25.443Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 225 | 2024-02-15T00:05:25.444Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 226 | 2024-02-15T00:05:25.444Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 227 | 2024-02-15T00:05:25.444Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 228 | 2024-02-15T00:05:25.444Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 229 | 2024-02-15T00:05:25.444Z | INFO | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/storage_monitor.rs:270
|
| 230 | 2024-02-15T00:05:28.318Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_switch
|
| 231 | 2024-02-15T00:05:28.613Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 29.644206ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 232 | 2024-02-15T00:05:28.812Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 147.265773ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 233 | 2024-02-15T00:05:29.009Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 174.367606ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 234 | 2024-02-15T00:05:29.366Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 265.789477ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 235 | 2024-02-15T00:05:29.776Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 947.14547ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 236 | 2024-02-15T00:05:30.957Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 891.315424ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 237 | 2024-02-15T00:05:32.083Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 764.141353ms
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 238 | 2024-02-15T00:05:32.989Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_switch") failed: Property not found. retry in 1.292792668s
file = illumos-utils/src/svc.rs:36
zone = oxz_switch
|
| 239 | 2024-02-15T00:05:35.460Z | INFO | SledAgent (ServiceManager): Ensuring tfportrear0_0/ll exists in zone
file = sled-agent/src/services.rs:1847
|
| 240 | 2024-02-15T00:05:35.681Z | INFO | SledAgent (ServiceManager): sled agent started
file = sled-agent/src/services.rs:978
underlay_address = fd00:1122:3344:101::1
|
| 241 | 2024-02-15T00:05:35.682Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]
file = internal-dns/src/resolver.rs:60
|
| 242 | 2024-02-15T00:05:35.738Z | INFO | SledAgent (dropshot (SledAgent)): listening
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195
local_addr = [fd00:1122:3344:101::1]:12345
|
| 243 | 2024-02-15T00:05:35.738Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 244 | 2024-02-15T00:05:35.738Z | INFO | SledAgent (SledAgentMain): Sled Agent loaded; recording configuration
file = sled-agent/src/bootstrap/server.rs:424
|
| 245 | 2024-02-15T00:05:35.738Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 246 | 2024-02-15T00:05:35.738Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:198
|
| 247 | 2024-02-15T00:05:35.738Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1199
retry_after = 344.215786ms
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 248 | 2024-02-15T00:05:35.738Z | INFO | SledAgent (SledAgentMain): Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp
file = common/src/ledger.rs:198
|
| 249 | 2024-02-15T00:05:35.739Z | INFO | SledAgent (HardwareMonitor): Sled Agent Started
file = sled-agent/src/hardware_monitor.rs:137
|
| 250 | 2024-02-15T00:05:35.739Z | INFO | SledAgent (HardwareMonitor): Checking current full hardware snapshot
file = sled-agent/src/hardware_monitor.rs:241
underlay_network_info = Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 192.168.1.30, infra_ip_last: 192.168.1.30, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100 }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: "qsfp0", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [] })))
|
| 251 | 2024-02-15T00:05:35.739Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
file = sled-agent/src/services.rs:3345
|
| 252 | 2024-02-15T00:05:35.739Z | INFO | SledAgent (SledAgentSprocketsServer): Connection closed
file = sled-agent/src/bootstrap/sprockets_server.rs:73
remote_addr = [fdb0:18c0:4d0d:a02a::1]:55221
|
| 253 | 2024-02-15T00:05:35.741Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
file = sled-agent/src/bootstrap/rss_handle.rs:108
peer = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 254 | 2024-02-15T00:05:35.741Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
file = sled-agent/src/bootstrap/rss_handle.rs:232
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 255 | 2024-02-15T00:05:35.742Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 256 | 2024-02-15T00:05:35.742Z | INFO | SledAgent (RSS): No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json
file = common/src/ledger.rs:187
|
| 257 | 2024-02-15T00:05:35.742Z | INFO | SledAgent (RSS): No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json
file = common/src/ledger.rs:187
|
| 258 | 2024-02-15T00:05:35.742Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 259 | 2024-02-15T00:05:35.742Z | INFO | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: dns error: proto error: io error: No route to host (os error 148)
file = sled-agent/src/sled_agent.rs:735
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 260 | 2024-02-15T00:05:35.778Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:40975
|
| 261 | 2024-02-15T00:05:35.778Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 262 | 2024-02-15T00:05:35.778Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 251
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:40975
req_id = 0d25f025-b8a2-40dc-beaa-0a2c961e624b
response_code = 200
uri = /zpools
|
| 263 | 2024-02-15T00:05:35.797Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 264 | 2024-02-15T00:05:35.809Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:34952
|
| 265 | 2024-02-15T00:05:35.809Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 48
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:34952
req_id = c472a43e-8e8f-4976-b3d8-ec672f841849
response_code = 200
uri = /sled-role
|
| 266 | 2024-02-15T00:05:35.812Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 267 | 2024-02-15T00:05:35.812Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp
file = common/src/ledger.rs:198
|
| 268 | 2024-02-15T00:05:35.812Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp
file = common/src/ledger.rs:198
|
| 269 | 2024-02-15T00:05:35.812Z | INFO | SledAgent (RSS): Service plan written to storage
file = sled-agent/src/rack_setup/plan/service.rs:748
|
| 270 | 2024-02-15T00:05:35.841Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 271 | 2024-02-15T00:05:35.842Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:52275
|
| 272 | 2024-02-15T00:05:35.842Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 273 | 2024-02-15T00:05:35.842Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 274 | 2024-02-15T00:05:35.894Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 275 | 2024-02-15T00:05:35.894Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 276 | 2024-02-15T00:05:35.919Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 277 | 2024-02-15T00:05:35.946Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 278 | 2024-02-15T00:05:35.946Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 279 | 2024-02-15T00:05:35.994Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 280 | 2024-02-15T00:05:35.994Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:187
|
| 281 | 2024-02-15T00:05:35.994Z | INFO | SledAgent (ServiceManager): No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:187
|
| 282 | 2024-02-15T00:05:35.994Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 283 | 2024-02-15T00:05:36.040Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 284 | 2024-02-15T00:05:36.040Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 285 | 2024-02-15T00:05:36.040Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 286 | 2024-02-15T00:05:36.087Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1199
retry_after = 556.361378ms
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 287 | 2024-02-15T00:05:36.094Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
file = illumos-utils/src/zone.rs:315
|
| 288 | 2024-02-15T00:05:36.098Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
file = illumos-utils/src/zone.rs:315
|
| 289 | 2024-02-15T00:05:36.102Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
file = illumos-utils/src/zone.rs:315
|
| 290 | 2024-02-15T00:05:36.114Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
file = illumos-utils/src/zone.rs:354
|
| 291 | 2024-02-15T00:05:36.119Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
file = illumos-utils/src/zone.rs:354
|
| 292 | 2024-02-15T00:05:36.123Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
file = illumos-utils/src/zone.rs:354
|
| 293 | 2024-02-15T00:05:36.131Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 294 | 2024-02-15T00:05:36.324Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 295 | 2024-02-15T00:05:36.565Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 296 | 2024-02-15T00:05:36.644Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1199
retry_after = 1.155028169s
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 297 | 2024-02-15T00:05:36.918Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 298 | 2024-02-15T00:05:37.364Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 299 | 2024-02-15T00:05:37.635Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 300 | 2024-02-15T00:05:37.800Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1199
retry_after = 1.148608041s
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 301 | 2024-02-15T00:05:37.947Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 302 | 2024-02-15T00:05:37.957Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 303 | 2024-02-15T00:05:37.963Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 304 | 2024-02-15T00:05:38.080Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249") failed: Property not found. retry in 68.369584ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 305 | 2024-02-15T00:05:38.088Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945") failed: Property not found. retry in 35.203624ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 306 | 2024-02-15T00:05:38.094Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796") failed: Property not found. retry in 55.926181ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 307 | 2024-02-15T00:05:38.131Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945") failed: Property not found. retry in 126.088873ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 308 | 2024-02-15T00:05:38.223Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249") failed: Property not found. retry in 59.827301ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 309 | 2024-02-15T00:05:38.223Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 310 | 2024-02-15T00:05:38.251Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796") failed: Property not found. retry in 62.611857ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 311 | 2024-02-15T00:05:38.275Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945") failed: Property not found. retry in 161.438468ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 312 | 2024-02-15T00:05:38.306Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249") failed: Property not found. retry in 297.309658ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 313 | 2024-02-15T00:05:38.365Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796") failed: Property not found. retry in 261.156789ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 314 | 2024-02-15T00:05:38.462Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945") failed: Property not found. retry in 312.206425ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 315 | 2024-02-15T00:05:38.619Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249") failed: Property not found. retry in 344.780926ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 316 | 2024-02-15T00:05:38.645Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796") failed: Property not found. retry in 556.665116ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 317 | 2024-02-15T00:05:38.661Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 318 | 2024-02-15T00:05:38.789Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945") failed: Property not found. retry in 646.969673ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 319 | 2024-02-15T00:05:38.951Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1199
retry_after = 4.667637681s
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 320 | 2024-02-15T00:05:38.974Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249") failed: Property not found. retry in 668.667693ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 321 | 2024-02-15T00:05:39.212Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796") failed: Property not found. retry in 739.773973ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 322 | 2024-02-15T00:05:39.287Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 323 | 2024-02-15T00:05:39.445Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945") failed: Property not found. retry in 914.675593ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 324 | 2024-02-15T00:05:39.653Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249") failed: Property not found. retry in 1.278064114s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 325 | 2024-02-15T00:05:39.780Z | INFO | SledAgent (ServiceManager): Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone
file = sled-agent/src/services.rs:1864
|
| 326 | 2024-02-15T00:05:39.780Z | INFO | SledAgent (ServiceManager): Adding bootstrap address
file = illumos-utils/src/running_zone.rs:626
zone = oxz_switch
|
| 327 | 2024-02-15T00:05:39.963Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796") failed: Property not found. retry in 506.871528ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 328 | 2024-02-15T00:05:40.074Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 329 | 2024-02-15T00:05:40.370Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945") failed: Property not found. retry in 1.07307941s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 330 | 2024-02-15T00:05:40.480Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796") failed: Property not found. retry in 915.165315ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 331 | 2024-02-15T00:05:40.939Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249") failed: Property not found. retry in 867.801139ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 332 | 2024-02-15T00:05:41.146Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 333 | 2024-02-15T00:05:41.404Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796") failed: Property not found. retry in 1.054033863s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 334 | 2024-02-15T00:05:41.450Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945") failed: Property not found. retry in 1.070626145s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 335 | 2024-02-15T00:05:41.815Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249") failed: Property not found. retry in 976.840296ms
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 336 | 2024-02-15T00:05:42.348Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 337 | 2024-02-15T00:05:42.468Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796") failed: Property not found. retry in 1.169854314s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 338 | 2024-02-15T00:05:42.530Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945") failed: Property not found. retry in 1.382001619s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 339 | 2024-02-15T00:05:42.801Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249") failed: Property not found. retry in 1.076636895s
file = illumos-utils/src/svc.rs:36
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 340 | 2024-02-15T00:05:43.079Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 341 | 2024-02-15T00:05:43.619Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"
file = sled-agent/src/sled_agent.rs:1199
retry_after = 7.40092571s
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 342 | 2024-02-15T00:05:43.681Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:1::1 exists
file = sled-agent/src/services.rs:1900
|
| 343 | 2024-02-15T00:05:43.681Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 344 | 2024-02-15T00:05:44.228Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:1::1 exists - OK
file = sled-agent/src/services.rs:1908
|
| 345 | 2024-02-15T00:05:44.228Z | INFO | SledAgent (ServiceManager): Zone not using gateway (even though underlay is up)
file = sled-agent/src/services.rs:1932
|
| 346 | 2024-02-15T00:05:44.310Z | INFO | SledAgent (ServiceManager): Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe4d:ce1d
file = sled-agent/src/services.rs:1871
|
| 347 | 2024-02-15T00:05:44.316Z | INFO | SledAgent (ServiceManager): Zone not using gateway (even though underlay is up)
file = sled-agent/src/services.rs:1932
|
| 348 | 2024-02-15T00:05:44.400Z | INFO | SledAgent (ServiceManager): Setting up internal-dns service
file = sled-agent/src/services.rs:2126
|
| 349 | 2024-02-15T00:05:44.437Z | INFO | SledAgent (ServiceManager): Setting up dendrite service
file = sled-agent/src/services.rs:2404
|
| 350 | 2024-02-15T00:05:44.671Z | INFO | SledAgent (ServiceManager): Setting up MGS service
file = sled-agent/src/services.rs:2283
|
| 351 | 2024-02-15T00:05:44.929Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:91
prefix = Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }
|
| 352 | 2024-02-15T00:05:44.929Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 353 | 2024-02-15T00:05:44.961Z | INFO | SledAgent (ServiceManager): Setting up wicketd service
file = sled-agent/src/services.rs:2318
|
| 354 | 2024-02-15T00:05:44.977Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:2::1 exists
file = sled-agent/src/services.rs:1900
|
| 355 | 2024-02-15T00:05:44.977Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:2::1, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 356 | 2024-02-15T00:05:45.161Z | INFO | SledAgent (ServiceManager): Setting up mgd service
file = sled-agent/src/services.rs:2585
|
| 357 | 2024-02-15T00:05:45.306Z | INFO | SledAgent (ServiceManager): Setting up mg-ddm service
file = sled-agent/src/services.rs:2590
|
| 358 | 2024-02-15T00:05:45.488Z | INFO | SledAgent (ServiceManager): Setting up tfport service
file = sled-agent/src/services.rs:2524
|
| 359 | 2024-02-15T00:05:45.529Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:2::1 exists - OK
file = sled-agent/src/services.rs:1908
|
| 360 | 2024-02-15T00:05:45.529Z | INFO | SledAgent (ServiceManager): Zone not using gateway (even though underlay is up)
file = sled-agent/src/services.rs:1932
|
| 361 | 2024-02-15T00:05:45.662Z | INFO | SledAgent (ServiceManager): Setting up Simulated SP service
file = sled-agent/src/services.rs:2312
|
| 362 | 2024-02-15T00:05:45.672Z | INFO | SledAgent (ServiceManager): Re-enabling running switch zone (new address)
file = sled-agent/src/services.rs:3597
new = [fd00:1122:3344:101::2, ::1]
old = [::1]
|
| 363 | 2024-02-15T00:05:45.672Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists
file = sled-agent/src/services.rs:3613
|
| 364 | 2024-02-15T00:05:45.672Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_switch
|
| 365 | 2024-02-15T00:05:45.713Z | INFO | SledAgent (ServiceManager): Setting up internal-dns service
file = sled-agent/src/services.rs:2126
|
| 366 | 2024-02-15T00:05:46.221Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists - OK
file = sled-agent/src/services.rs:3621
|
| 367 | 2024-02-15T00:05:46.228Z | INFO | SledAgent (ServiceManager): configuring dendrite zone
file = sled-agent/src/services.rs:3677
|
| 368 | 2024-02-15T00:05:46.269Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:3::1 exists
file = sled-agent/src/services.rs:1900
|
| 369 | 2024-02-15T00:05:46.269Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:3::1, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 370 | 2024-02-15T00:05:46.422Z | INFO | SledAgent (ServiceManager): configuring wicketd
file = sled-agent/src/services.rs:3717
rack_subnet = fd00:1122:3344::
|
| 371 | 2024-02-15T00:05:46.472Z | INFO | SledAgent (ServiceManager): Determining physical location of our switch zone at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:360
|
| 372 | 2024-02-15T00:05:46.504Z | WARN | SledAgent (ServiceManager): Failed to get switch ID from MGS (retrying in 62.437847ms)
error = Error Response: status: 503 Service Unavailable; headers: {"content-type": "application/json", "x-request-id": "0a20033a-fe8e-45b3-b3a7-bff682b01c9e", "content-length": "152", "date": "Thu, 15 Feb 2024 00:05:46 GMT"}; value: Error { error_code: Some("DiscoveryNotYetComplete"), message: "discovery process not yet complete", request_id: "0a20033a-fe8e-45b3-b3a7-bff682b01c9e" }
file = sled-agent/src/bootstrap/early_networking.rs:379
|
| 373 | 2024-02-15T00:05:46.568Z | WARN | SledAgent (ServiceManager): Failed to get switch ID from MGS (retrying in 118.5065ms)
error = Error Response: status: 503 Service Unavailable; headers: {"content-type": "application/json", "x-request-id": "36b656f1-c86d-418b-b713-7b9430fe9102", "content-length": "152", "date": "Thu, 15 Feb 2024 00:05:46 GMT"}; value: Error { error_code: Some("DiscoveryNotYetComplete"), message: "discovery process not yet complete", request_id: "36b656f1-c86d-418b-b713-7b9430fe9102" }
file = sled-agent/src/bootstrap/early_networking.rs:379
|
| 374 | 2024-02-15T00:05:46.688Z | WARN | SledAgent (ServiceManager): Failed to get switch ID from MGS (retrying in 184.549651ms)
error = Error Response: status: 503 Service Unavailable; headers: {"content-type": "application/json", "x-request-id": "a5921b3a-11f0-4df3-949f-4fb7e9927700", "content-length": "152", "date": "Thu, 15 Feb 2024 00:05:46 GMT"}; value: Error { error_code: Some("DiscoveryNotYetComplete"), message: "discovery process not yet complete", request_id: "a5921b3a-11f0-4df3-949f-4fb7e9927700" }
file = sled-agent/src/bootstrap/early_networking.rs:379
|
| 375 | 2024-02-15T00:05:46.821Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:3::1 exists - OK
file = sled-agent/src/services.rs:1908
|
| 376 | 2024-02-15T00:05:46.821Z | INFO | SledAgent (ServiceManager): Zone not using gateway (even though underlay is up)
file = sled-agent/src/services.rs:1932
|
| 377 | 2024-02-15T00:05:46.874Z | INFO | SledAgent (ServiceManager): Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:408
|
| 378 | 2024-02-15T00:05:46.906Z | INFO | SledAgent (ServiceManager): Building Port Configuration
file = sled-agent/src/bootstrap/early_networking.rs:567
|
| 379 | 2024-02-15T00:05:46.906Z | INFO | SledAgent (ServiceManager): Checking dendrite uptime
file = sled-agent/src/bootstrap/early_networking.rs:609
|
| 380 | 2024-02-15T00:05:46.907Z | INFO | SledAgent (ServiceManager): Dendrite online
file = sled-agent/src/bootstrap/early_networking.rs:612
uptime = 2
|
| 381 | 2024-02-15T00:05:46.907Z | INFO | SledAgent (ServiceManager): Configuring default uplink on switch
config = PortSettings {\n links: {\n "0": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n v4_routes: {},\n v6_routes: {},\n}
file = sled-agent/src/bootstrap/early_networking.rs:430
|
| 382 | 2024-02-15T00:05:46.982Z | INFO | SledAgent (StorageManager): Received DisksChanged({})
file = sled-storage/src/manager.rs:346
|
| 383 | 2024-02-15T00:05:46.996Z | INFO | SledAgent (ServiceManager): Setting up internal-dns service
file = sled-agent/src/services.rs:2126
|
| 384 | 2024-02-15T00:05:47.477Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:91
prefix = Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }
|
| 385 | 2024-02-15T00:05:47.527Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_internal_dns_a3944496-e534-42b6-afe3-78830d746249
|
| 386 | 2024-02-15T00:05:47.527Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_internal_dns_eaa4434d-a6bd-48f6-beb2-151404cd7796
|
| 387 | 2024-02-15T00:05:47.527Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_internal_dns_5e4f48e8-b987-45c0-8c5b-523ef542f945
|
| 388 | 2024-02-15T00:05:47.527Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:198
|
| 389 | 2024-02-15T00:05:47.527Z | INFO | SledAgent (BootstrapAgentStartup): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
file = clients/ddm-admin-client/src/lib.rs:91
prefix = Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }
|
| 390 | 2024-02-15T00:05:47.527Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:198
|
| 391 | 2024-02-15T00:05:47.528Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 11685984
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:52275
req_id = 1abc9b2a-e982-4d0c-8c89-87896cc73b91
response_code = 204
uri = /omicron-zones
|
| 392 | 2024-02-15T00:05:47.529Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:1::1]:5353
file = sled-agent/src/rack_setup/service.rs:417
|
| 393 | 2024-02-15T00:05:47.578Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:2::1]:5353
file = sled-agent/src/rack_setup/service.rs:417
|
| 394 | 2024-02-15T00:05:47.623Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:3::1]:5353
file = sled-agent/src/rack_setup/service.rs:417
|
| 395 | 2024-02-15T00:05:48.790Z | INFO | SledAgent (RSS): Configured all DNS servers
file = sled-agent/src/rack_setup/service.rs:453
|
| 396 | 2024-02-15T00:05:48.790Z | INFO | SledAgent (RSS): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:254
|
| 397 | 2024-02-15T00:05:48.822Z | INFO | SledAgent (RSS): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:271
|
| 398 | 2024-02-15T00:05:48.824Z | INFO | SledAgent (RSS): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:296
|
| 399 | 2024-02-15T00:05:48.824Z | INFO | SledAgent (RSS): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:188
|
| 400 | 2024-02-15T00:05:48.852Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 401 | 2024-02-15T00:05:48.852Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:39411
|
| 402 | 2024-02-15T00:05:48.853Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 403 | 2024-02-15T00:05:48.853Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 404 | 2024-02-15T00:05:48.873Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 405 | 2024-02-15T00:05:48.873Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 406 | 2024-02-15T00:05:48.890Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 407 | 2024-02-15T00:05:48.890Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 408 | 2024-02-15T00:05:48.908Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 409 | 2024-02-15T00:05:48.908Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 410 | 2024-02-15T00:05:48.908Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 411 | 2024-02-15T00:05:48.909Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 412 | 2024-02-15T00:05:48.920Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:254
|
| 413 | 2024-02-15T00:05:48.953Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:271
|
| 414 | 2024-02-15T00:05:48.954Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:296
|
| 415 | 2024-02-15T00:05:48.954Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:188
|
| 416 | 2024-02-15T00:05:48.999Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte0", _ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 214, 124]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: "vopte0" } }
|
| 417 | 2024-02-15T00:05:48.999Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1268
zone_type = ntp
|
| 418 | 2024-02-15T00:05:49.000Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:98
|
| 419 | 2024-02-15T00:05:49.000Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:105
|
| 420 | 2024-02-15T00:05:49.000Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.169,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:133
|
| 421 | 2024-02-15T00:05:49.002Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 422 | 2024-02-15T00:05:49.024Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
file = illumos-utils/src/zone.rs:315
|
| 423 | 2024-02-15T00:05:49.041Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
file = illumos-utils/src/zone.rs:354
|
| 424 | 2024-02-15T00:05:50.060Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 425 | 2024-02-15T00:05:50.190Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7") failed: Property not found. retry in 73.795842ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 426 | 2024-02-15T00:05:50.260Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:61982
|
| 427 | 2024-02-15T00:05:50.260Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 63
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:61982
req_id = 3caffbb0-793e-4a72-9b08-dfbe578063d3
response_code = 200
uri = /baseboard
|
| 428 | 2024-02-15T00:05:50.279Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7") failed: Property not found. retry in 137.526272ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 429 | 2024-02-15T00:05:50.431Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7") failed: Property not found. retry in 130.889811ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 430 | 2024-02-15T00:05:50.572Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7") failed: Property not found. retry in 373.882285ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 431 | 2024-02-15T00:05:50.955Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7") failed: Property not found. retry in 790.343089ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 432 | 2024-02-15T00:05:51.757Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7") failed: Property not found. retry in 1.316745307s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 433 | 2024-02-15T00:05:53.082Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7") failed: Property not found. retry in 1.026600876s
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 434 | 2024-02-15T00:05:54.118Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7") failed: Property not found. retry in 684.52365ms
file = illumos-utils/src/svc.rs:36
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 435 | 2024-02-15T00:05:54.845Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::1b exists
file = sled-agent/src/services.rs:1900
|
| 436 | 2024-02-15T00:05:54.845Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::1b, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 437 | 2024-02-15T00:05:55.397Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::1b exists - OK
file = sled-agent/src/services.rs:1908
|
| 438 | 2024-02-15T00:05:55.397Z | INFO | SledAgent (ServiceManager): Zone using sled underlay as gateway
file = sled-agent/src/services.rs:1927
|
| 439 | 2024-02-15T00:05:55.591Z | INFO | SledAgent (ServiceManager): Set up NTP service boundary=true, Servers=["0.pool.ntp.org"]
file = sled-agent/src/services.rs:2209
|
| 440 | 2024-02-15T00:05:55.618Z | INFO | SledAgent (ServiceManager): Ensuring address for OPTE port
file = illumos-utils/src/running_zone.rs:653
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 441 | 2024-02-15T00:06:01.257Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7
|
| 442 | 2024-02-15T00:06:01.257Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:198
|
| 443 | 2024-02-15T00:06:01.257Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:198
|
| 444 | 2024-02-15T00:06:01.257Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 12404847
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:39411
req_id = 02b951fc-5bbc-4c7f-91ca-516be61af134
response_code = 204
uri = /omicron-zones
|
| 445 | 2024-02-15T00:06:01.260Z | INFO | SledAgent (RSS): Waiting for rack time synchronization
file = sled-agent/src/rack_setup/service.rs:494
|
| 446 | 2024-02-15T00:06:01.290Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 447 | 2024-02-15T00:06:01.290Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:53300
|
| 448 | 2024-02-15T00:06:01.298Z | INFO | SledAgent (ServiceManager): chronyc command failed: Error running command in zone 'oxz_ntp_06e1927a-c14e-4e46-8691-4b4ad6c57bc7': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon
stderr:
file = sled-agent/src/services.rs:3331
|
| 449 | 2024-02-15T00:06:01.299Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = NTP zone not ready
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 8483
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:53300
req_id = 8afc686b-e55c-41ed-a08d-dbb5abb16738
response_code = 500
uri = /timesync
|
| 450 | 2024-02-15T00:06:01.301Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 451 | 2024-02-15T00:06:01.431Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 452 | 2024-02-15T00:06:01.431Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:36721
|
| 453 | 2024-02-15T00:06:01.439Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8441
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:36721
req_id = 6d2aee67-dd90-4945-9b4b-c3f04f1fb19e
response_code = 200
uri = /timesync
|
| 454 | 2024-02-15T00:06:01.442Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 455 | 2024-02-15T00:06:01.442Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 456 | 2024-02-15T00:06:01.557Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 457 | 2024-02-15T00:06:01.558Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:63723
|
| 458 | 2024-02-15T00:06:01.566Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8533
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:63723
req_id = 48772b44-2a2c-4ff8-bdbf-dfaefef3fbca
response_code = 200
uri = /timesync
|
| 459 | 2024-02-15T00:06:01.570Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 460 | 2024-02-15T00:06:01.570Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 461 | 2024-02-15T00:06:01.751Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 462 | 2024-02-15T00:06:01.751Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:43421
|
| 463 | 2024-02-15T00:06:01.759Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8008
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:43421
req_id = 5a2e1b08-48f4-4e56-ac97-d9e68f76f75d
response_code = 200
uri = /timesync
|
| 464 | 2024-02-15T00:06:01.762Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 465 | 2024-02-15T00:06:01.762Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 466 | 2024-02-15T00:06:01.977Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 467 | 2024-02-15T00:06:01.977Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:53916
|
| 468 | 2024-02-15T00:06:01.986Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8216
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:53916
req_id = 8e3a23a3-11d7-41a2-a065-df027a9ee0e9
response_code = 200
uri = /timesync
|
| 469 | 2024-02-15T00:06:01.989Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 470 | 2024-02-15T00:06:01.989Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 471 | 2024-02-15T00:06:02.272Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 472 | 2024-02-15T00:06:02.272Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:45950
|
| 473 | 2024-02-15T00:06:02.282Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 9123
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:45950
req_id = bf99c6aa-d532-4fdf-b814-45ce202895fb
response_code = 200
uri = /timesync
|
| 474 | 2024-02-15T00:06:02.284Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 475 | 2024-02-15T00:06:02.284Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 476 | 2024-02-15T00:06:02.538Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 477 | 2024-02-15T00:06:02.538Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:43200
|
| 478 | 2024-02-15T00:06:02.547Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8674
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:43200
req_id = 8ff96eac-f24d-4437-abc1-31f63bdb93e1
response_code = 200
uri = /timesync
|
| 479 | 2024-02-15T00:06:02.551Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 480 | 2024-02-15T00:06:02.551Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 481 | 2024-02-15T00:06:02.929Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 482 | 2024-02-15T00:06:02.930Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:41938
|
| 483 | 2024-02-15T00:06:02.938Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 7979
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:41938
req_id = 25dca2e1-3367-4845-8917-2bb7e59e571c
response_code = 200
uri = /timesync
|
| 484 | 2024-02-15T00:06:02.941Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 485 | 2024-02-15T00:06:02.941Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 486 | 2024-02-15T00:06:03.447Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 487 | 2024-02-15T00:06:03.447Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:38247
|
| 488 | 2024-02-15T00:06:03.457Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 9443
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:38247
req_id = 54289073-9430-4acf-a12e-aa5d178c8685
response_code = 200
uri = /timesync
|
| 489 | 2024-02-15T00:06:03.459Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 490 | 2024-02-15T00:06:03.459Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 491 | 2024-02-15T00:06:03.856Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 492 | 2024-02-15T00:06:03.857Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:58137
|
| 493 | 2024-02-15T00:06:03.866Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8819
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:58137
req_id = 6dde0ebf-e651-493c-872d-7dc6ab23bb57
response_code = 200
uri = /timesync
|
| 494 | 2024-02-15T00:06:03.869Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 495 | 2024-02-15T00:06:03.869Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 496 | 2024-02-15T00:06:04.482Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 497 | 2024-02-15T00:06:04.482Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:47240
|
| 498 | 2024-02-15T00:06:04.489Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 7494
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:47240
req_id = 1f2b9fa0-8d0b-4116-babb-f80e95e99b6c
response_code = 200
uri = /timesync
|
| 499 | 2024-02-15T00:06:04.491Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 500 | 2024-02-15T00:06:04.491Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 501 | 2024-02-15T00:06:05.222Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 502 | 2024-02-15T00:06:05.222Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:56625
|
| 503 | 2024-02-15T00:06:05.231Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8564
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:56625
req_id = 30914209-b0a8-4168-8b48-e8d308578f64
response_code = 200
uri = /timesync
|
| 504 | 2024-02-15T00:06:05.233Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 505 | 2024-02-15T00:06:05.233Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 506 | 2024-02-15T00:06:05.934Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 507 | 2024-02-15T00:06:05.934Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:59941
|
| 508 | 2024-02-15T00:06:05.942Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8393
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:59941
req_id = 106bba53-9f53-499b-9af6-2b898346a6f2
response_code = 200
uri = /timesync
|
| 509 | 2024-02-15T00:06:05.945Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 510 | 2024-02-15T00:06:05.945Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 511 | 2024-02-15T00:06:07.479Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 512 | 2024-02-15T00:06:07.479Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:55126
|
| 513 | 2024-02-15T00:06:07.483Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:776
|
| 514 | 2024-02-15T00:06:07.487Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8039
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:55126
req_id = 1ac36ca6-4ed0-4349-a8a2-7826fac8d2b7
response_code = 200
uri = /timesync
|
| 515 | 2024-02-15T00:06:07.490Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707955559.8823986, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 516 | 2024-02-15T00:06:07.490Z | WARN | SledAgent (RSS): Time is not yet synchronized
error = "Time is synchronized on 0/1 sleds"
file = sled-agent/src/rack_setup/service.rs:524
|
| 517 | 2024-02-15T00:06:08.409Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
file = sled-agent/src/rack_setup/service.rs:474
|
| 518 | 2024-02-15T00:06:08.410Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:48944
|
| 519 | 2024-02-15T00:06:08.413Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:776
|
| 520 | 2024-02-15T00:06:08.432Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 21978
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:48944
req_id = 8379a709-1ebd-4ec6-8be7-49c61235ac22
response_code = 200
uri = /timesync
|
| 521 | 2024-02-15T00:06:08.435Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380539, ip_addr: 162.159.200.123, stratum: 4, ref_time: 1707955567.670102, correction: 0.0 }
file = sled-agent/src/rack_setup/service.rs:502
|
| 522 | 2024-02-15T00:06:08.435Z | INFO | SledAgent (RSS): Finished setting up Internal DNS and NTP
file = sled-agent/src/rack_setup/service.rs:1039
|
| 523 | 2024-02-15T00:06:08.464Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 524 | 2024-02-15T00:06:08.464Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:61545
|
| 525 | 2024-02-15T00:06:08.465Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 526 | 2024-02-15T00:06:08.465Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 527 | 2024-02-15T00:06:08.485Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 528 | 2024-02-15T00:06:08.485Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 529 | 2024-02-15T00:06:08.503Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 530 | 2024-02-15T00:06:08.503Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 531 | 2024-02-15T00:06:08.521Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 532 | 2024-02-15T00:06:08.521Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 533 | 2024-02-15T00:06:08.572Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 534 | 2024-02-15T00:06:08.572Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 535 | 2024-02-15T00:06:08.624Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 536 | 2024-02-15T00:06:08.624Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 537 | 2024-02-15T00:06:08.676Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 538 | 2024-02-15T00:06:08.676Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 539 | 2024-02-15T00:06:08.728Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 540 | 2024-02-15T00:06:08.728Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 541 | 2024-02-15T00:06:08.779Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 542 | 2024-02-15T00:06:08.779Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 543 | 2024-02-15T00:06:08.779Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 544 | 2024-02-15T00:06:08.779Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 545 | 2024-02-15T00:06:08.812Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 546 | 2024-02-15T00:06:08.812Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 547 | 2024-02-15T00:06:08.812Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 548 | 2024-02-15T00:06:08.812Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 549 | 2024-02-15T00:06:08.812Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 550 | 2024-02-15T00:06:08.812Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 347435
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = edb9ffcc-9b38-45cd-94cc-028ce84b55d0
response_code = 500
uri = /omicron-zones
|
| 551 | 2024-02-15T00:06:08.812Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "edb9ffcc-9b38-45cd-94cc-028ce84b55d0", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:08 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "edb9ffcc-9b38-45cd-94cc-028ce84b55d0" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 138.025882ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 552 | 2024-02-15T00:06:08.951Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 553 | 2024-02-15T00:06:08.952Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 554 | 2024-02-15T00:06:08.952Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 555 | 2024-02-15T00:06:08.970Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 556 | 2024-02-15T00:06:08.970Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 557 | 2024-02-15T00:06:08.988Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 558 | 2024-02-15T00:06:08.988Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 559 | 2024-02-15T00:06:09.006Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 560 | 2024-02-15T00:06:09.006Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 561 | 2024-02-15T00:06:09.025Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 562 | 2024-02-15T00:06:09.025Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 563 | 2024-02-15T00:06:09.042Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 564 | 2024-02-15T00:06:09.042Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 565 | 2024-02-15T00:06:09.060Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 566 | 2024-02-15T00:06:09.060Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 567 | 2024-02-15T00:06:09.077Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 568 | 2024-02-15T00:06:09.077Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 569 | 2024-02-15T00:06:09.095Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 570 | 2024-02-15T00:06:09.095Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 571 | 2024-02-15T00:06:09.095Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 572 | 2024-02-15T00:06:09.095Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 573 | 2024-02-15T00:06:09.130Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 574 | 2024-02-15T00:06:09.130Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 575 | 2024-02-15T00:06:09.130Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 576 | 2024-02-15T00:06:09.130Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 577 | 2024-02-15T00:06:09.130Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 578 | 2024-02-15T00:06:09.130Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 177919
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 77ff47e8-36b8-411f-89f9-d8eb1fe33e16
response_code = 500
uri = /omicron-zones
|
| 579 | 2024-02-15T00:06:09.130Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "77ff47e8-36b8-411f-89f9-d8eb1fe33e16", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:08 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "77ff47e8-36b8-411f-89f9-d8eb1fe33e16" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 102.961201ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 580 | 2024-02-15T00:06:09.235Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 581 | 2024-02-15T00:06:09.235Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 582 | 2024-02-15T00:06:09.235Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 583 | 2024-02-15T00:06:09.255Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 584 | 2024-02-15T00:06:09.255Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 585 | 2024-02-15T00:06:09.272Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 586 | 2024-02-15T00:06:09.272Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 587 | 2024-02-15T00:06:09.290Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 588 | 2024-02-15T00:06:09.290Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 589 | 2024-02-15T00:06:09.308Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 590 | 2024-02-15T00:06:09.308Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 591 | 2024-02-15T00:06:09.326Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 592 | 2024-02-15T00:06:09.326Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 593 | 2024-02-15T00:06:09.344Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 594 | 2024-02-15T00:06:09.344Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 595 | 2024-02-15T00:06:09.362Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 596 | 2024-02-15T00:06:09.362Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 597 | 2024-02-15T00:06:09.380Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 598 | 2024-02-15T00:06:09.380Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 599 | 2024-02-15T00:06:09.380Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 600 | 2024-02-15T00:06:09.380Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 601 | 2024-02-15T00:06:09.413Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 602 | 2024-02-15T00:06:09.413Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 603 | 2024-02-15T00:06:09.413Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 604 | 2024-02-15T00:06:09.413Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 605 | 2024-02-15T00:06:09.413Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 606 | 2024-02-15T00:06:09.413Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 177256
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 428475d9-f4b2-4afa-b141-7e173f2f14ee
response_code = 500
uri = /omicron-zones
|
| 607 | 2024-02-15T00:06:09.413Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "428475d9-f4b2-4afa-b141-7e173f2f14ee", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:08 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "428475d9-f4b2-4afa-b141-7e173f2f14ee" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 162.528069ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 608 | 2024-02-15T00:06:09.577Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 609 | 2024-02-15T00:06:09.577Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 610 | 2024-02-15T00:06:09.577Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 611 | 2024-02-15T00:06:09.596Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 612 | 2024-02-15T00:06:09.596Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 613 | 2024-02-15T00:06:09.614Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 614 | 2024-02-15T00:06:09.614Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 615 | 2024-02-15T00:06:09.632Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 616 | 2024-02-15T00:06:09.632Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 617 | 2024-02-15T00:06:09.650Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 618 | 2024-02-15T00:06:09.650Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 619 | 2024-02-15T00:06:09.668Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 620 | 2024-02-15T00:06:09.668Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 621 | 2024-02-15T00:06:09.686Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 622 | 2024-02-15T00:06:09.686Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 623 | 2024-02-15T00:06:09.704Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 624 | 2024-02-15T00:06:09.704Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 625 | 2024-02-15T00:06:09.722Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 626 | 2024-02-15T00:06:09.722Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 627 | 2024-02-15T00:06:09.723Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 628 | 2024-02-15T00:06:09.723Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 629 | 2024-02-15T00:06:09.755Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 630 | 2024-02-15T00:06:09.755Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 631 | 2024-02-15T00:06:09.755Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 632 | 2024-02-15T00:06:09.755Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 633 | 2024-02-15T00:06:09.755Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 634 | 2024-02-15T00:06:09.756Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 177893
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 7c063265-9072-4258-8cfb-38a840a42ed2
response_code = 500
uri = /omicron-zones
|
| 635 | 2024-02-15T00:06:09.756Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "7c063265-9072-4258-8cfb-38a840a42ed2", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:08 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "7c063265-9072-4258-8cfb-38a840a42ed2" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 184.045982ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 636 | 2024-02-15T00:06:09.942Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 637 | 2024-02-15T00:06:09.943Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 638 | 2024-02-15T00:06:09.943Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 639 | 2024-02-15T00:06:09.962Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 640 | 2024-02-15T00:06:09.962Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 641 | 2024-02-15T00:06:09.980Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 642 | 2024-02-15T00:06:09.980Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 643 | 2024-02-15T00:06:09.999Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 644 | 2024-02-15T00:06:09.999Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 645 | 2024-02-15T00:06:10.021Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 646 | 2024-02-15T00:06:10.021Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 647 | 2024-02-15T00:06:10.040Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 648 | 2024-02-15T00:06:10.040Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 649 | 2024-02-15T00:06:10.059Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 650 | 2024-02-15T00:06:10.059Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 651 | 2024-02-15T00:06:10.078Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 652 | 2024-02-15T00:06:10.078Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 653 | 2024-02-15T00:06:10.096Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 654 | 2024-02-15T00:06:10.096Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 655 | 2024-02-15T00:06:10.096Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 656 | 2024-02-15T00:06:10.096Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 657 | 2024-02-15T00:06:10.129Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 658 | 2024-02-15T00:06:10.129Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 659 | 2024-02-15T00:06:10.129Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 660 | 2024-02-15T00:06:10.129Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 661 | 2024-02-15T00:06:10.129Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 662 | 2024-02-15T00:06:10.129Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 186089
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 995eef98-ab01-48bd-9219-66f4ef19f1f1
response_code = 500
uri = /omicron-zones
|
| 663 | 2024-02-15T00:06:10.129Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "995eef98-ab01-48bd-9219-66f4ef19f1f1", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:10 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "995eef98-ab01-48bd-9219-66f4ef19f1f1" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 106.190662ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 664 | 2024-02-15T00:06:10.236Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 665 | 2024-02-15T00:06:10.238Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 666 | 2024-02-15T00:06:10.238Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 667 | 2024-02-15T00:06:10.258Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 668 | 2024-02-15T00:06:10.258Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 669 | 2024-02-15T00:06:10.275Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 670 | 2024-02-15T00:06:10.276Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 671 | 2024-02-15T00:06:10.293Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 672 | 2024-02-15T00:06:10.293Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 673 | 2024-02-15T00:06:10.311Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 674 | 2024-02-15T00:06:10.312Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 675 | 2024-02-15T00:06:10.330Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 676 | 2024-02-15T00:06:10.330Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 677 | 2024-02-15T00:06:10.348Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 678 | 2024-02-15T00:06:10.348Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 679 | 2024-02-15T00:06:10.366Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 680 | 2024-02-15T00:06:10.366Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 681 | 2024-02-15T00:06:10.384Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 682 | 2024-02-15T00:06:10.384Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 683 | 2024-02-15T00:06:10.384Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 684 | 2024-02-15T00:06:10.384Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 685 | 2024-02-15T00:06:10.416Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 686 | 2024-02-15T00:06:10.416Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 687 | 2024-02-15T00:06:10.416Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 688 | 2024-02-15T00:06:10.416Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 689 | 2024-02-15T00:06:10.416Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 690 | 2024-02-15T00:06:10.417Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 179277
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 3bbf0581-5ce9-4f78-84ea-74666d41af36
response_code = 500
uri = /omicron-zones
|
| 691 | 2024-02-15T00:06:10.417Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3bbf0581-5ce9-4f78-84ea-74666d41af36", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:10 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3bbf0581-5ce9-4f78-84ea-74666d41af36" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 252.492621ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 692 | 2024-02-15T00:06:10.671Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 693 | 2024-02-15T00:06:10.672Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 694 | 2024-02-15T00:06:10.672Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 695 | 2024-02-15T00:06:10.691Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 696 | 2024-02-15T00:06:10.691Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 697 | 2024-02-15T00:06:10.709Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 698 | 2024-02-15T00:06:10.709Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 699 | 2024-02-15T00:06:10.726Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 700 | 2024-02-15T00:06:10.726Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 701 | 2024-02-15T00:06:10.744Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 702 | 2024-02-15T00:06:10.744Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 703 | 2024-02-15T00:06:10.763Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 704 | 2024-02-15T00:06:10.763Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 705 | 2024-02-15T00:06:10.781Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 706 | 2024-02-15T00:06:10.781Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 707 | 2024-02-15T00:06:10.799Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 708 | 2024-02-15T00:06:10.799Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 709 | 2024-02-15T00:06:10.817Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 710 | 2024-02-15T00:06:10.817Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 711 | 2024-02-15T00:06:10.818Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 712 | 2024-02-15T00:06:10.818Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 713 | 2024-02-15T00:06:10.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 714 | 2024-02-15T00:06:10.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 715 | 2024-02-15T00:06:10.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 716 | 2024-02-15T00:06:10.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 717 | 2024-02-15T00:06:10.850Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 718 | 2024-02-15T00:06:10.850Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 178706
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 9b3b328b-7d9e-4569-8eca-6678ed9f5848
response_code = 500
uri = /omicron-zones
|
| 719 | 2024-02-15T00:06:10.851Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "9b3b328b-7d9e-4569-8eca-6678ed9f5848", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:10 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "9b3b328b-7d9e-4569-8eca-6678ed9f5848" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 270.849412ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 720 | 2024-02-15T00:06:11.123Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 721 | 2024-02-15T00:06:11.123Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 722 | 2024-02-15T00:06:11.123Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 723 | 2024-02-15T00:06:11.142Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 724 | 2024-02-15T00:06:11.142Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 725 | 2024-02-15T00:06:11.161Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 726 | 2024-02-15T00:06:11.161Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 727 | 2024-02-15T00:06:11.179Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 728 | 2024-02-15T00:06:11.179Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 729 | 2024-02-15T00:06:11.197Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 730 | 2024-02-15T00:06:11.197Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 731 | 2024-02-15T00:06:11.216Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 732 | 2024-02-15T00:06:11.216Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 733 | 2024-02-15T00:06:11.234Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 734 | 2024-02-15T00:06:11.235Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 735 | 2024-02-15T00:06:11.252Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 736 | 2024-02-15T00:06:11.252Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 737 | 2024-02-15T00:06:11.270Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 738 | 2024-02-15T00:06:11.270Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 739 | 2024-02-15T00:06:11.270Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 740 | 2024-02-15T00:06:11.270Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 741 | 2024-02-15T00:06:11.302Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 742 | 2024-02-15T00:06:11.302Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 743 | 2024-02-15T00:06:11.302Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 744 | 2024-02-15T00:06:11.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 745 | 2024-02-15T00:06:11.303Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 746 | 2024-02-15T00:06:11.303Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 179222
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 06fa24a4-1a7e-4c90-adbd-a4ef6b232caf
response_code = 500
uri = /omicron-zones
|
| 747 | 2024-02-15T00:06:11.303Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "06fa24a4-1a7e-4c90-adbd-a4ef6b232caf", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:11 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "06fa24a4-1a7e-4c90-adbd-a4ef6b232caf" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 511.117192ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 748 | 2024-02-15T00:06:11.817Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 749 | 2024-02-15T00:06:11.818Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 750 | 2024-02-15T00:06:11.818Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 751 | 2024-02-15T00:06:11.838Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 752 | 2024-02-15T00:06:11.838Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 753 | 2024-02-15T00:06:11.856Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 754 | 2024-02-15T00:06:11.856Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 755 | 2024-02-15T00:06:11.875Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 756 | 2024-02-15T00:06:11.875Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 757 | 2024-02-15T00:06:11.893Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 758 | 2024-02-15T00:06:11.893Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 759 | 2024-02-15T00:06:11.912Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 760 | 2024-02-15T00:06:11.912Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 761 | 2024-02-15T00:06:11.930Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 762 | 2024-02-15T00:06:11.930Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 763 | 2024-02-15T00:06:11.948Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 764 | 2024-02-15T00:06:11.948Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 765 | 2024-02-15T00:06:11.966Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 766 | 2024-02-15T00:06:11.966Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 767 | 2024-02-15T00:06:11.966Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 768 | 2024-02-15T00:06:11.967Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 769 | 2024-02-15T00:06:11.999Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 770 | 2024-02-15T00:06:11.999Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 771 | 2024-02-15T00:06:11.999Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 772 | 2024-02-15T00:06:11.999Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 773 | 2024-02-15T00:06:11.999Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 774 | 2024-02-15T00:06:12.000Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 181781
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = e8c131d1-895f-41ad-b910-57819b424fdd
response_code = 500
uri = /omicron-zones
|
| 775 | 2024-02-15T00:06:12.000Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e8c131d1-895f-41ad-b910-57819b424fdd", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:11 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e8c131d1-895f-41ad-b910-57819b424fdd" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 477.823174ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 776 | 2024-02-15T00:06:12.481Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 777 | 2024-02-15T00:06:12.481Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 778 | 2024-02-15T00:06:12.481Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 779 | 2024-02-15T00:06:12.501Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 780 | 2024-02-15T00:06:12.501Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 781 | 2024-02-15T00:06:12.519Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 782 | 2024-02-15T00:06:12.519Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 783 | 2024-02-15T00:06:12.536Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 784 | 2024-02-15T00:06:12.536Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 785 | 2024-02-15T00:06:12.554Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 786 | 2024-02-15T00:06:12.554Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 787 | 2024-02-15T00:06:12.572Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 788 | 2024-02-15T00:06:12.572Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 789 | 2024-02-15T00:06:12.589Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 790 | 2024-02-15T00:06:12.589Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 791 | 2024-02-15T00:06:12.608Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 792 | 2024-02-15T00:06:12.608Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 793 | 2024-02-15T00:06:12.626Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 794 | 2024-02-15T00:06:12.626Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 795 | 2024-02-15T00:06:12.626Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 796 | 2024-02-15T00:06:12.626Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 797 | 2024-02-15T00:06:12.657Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 798 | 2024-02-15T00:06:12.657Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 799 | 2024-02-15T00:06:12.657Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 800 | 2024-02-15T00:06:12.657Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 801 | 2024-02-15T00:06:12.657Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 802 | 2024-02-15T00:06:12.658Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 176018
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 597649ff-c3d1-4fb3-bda2-3e6f06341379
response_code = 500
uri = /omicron-zones
|
| 803 | 2024-02-15T00:06:12.658Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "597649ff-c3d1-4fb3-bda2-3e6f06341379", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:11 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "597649ff-c3d1-4fb3-bda2-3e6f06341379" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 769.971992ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 804 | 2024-02-15T00:06:13.431Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 805 | 2024-02-15T00:06:13.432Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 806 | 2024-02-15T00:06:13.432Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 807 | 2024-02-15T00:06:13.452Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 808 | 2024-02-15T00:06:13.452Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 809 | 2024-02-15T00:06:13.470Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 810 | 2024-02-15T00:06:13.470Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 811 | 2024-02-15T00:06:13.488Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 812 | 2024-02-15T00:06:13.488Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 813 | 2024-02-15T00:06:13.507Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 814 | 2024-02-15T00:06:13.507Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 815 | 2024-02-15T00:06:13.526Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 816 | 2024-02-15T00:06:13.526Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 817 | 2024-02-15T00:06:13.544Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 818 | 2024-02-15T00:06:13.544Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 819 | 2024-02-15T00:06:13.563Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 820 | 2024-02-15T00:06:13.563Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 821 | 2024-02-15T00:06:13.581Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 822 | 2024-02-15T00:06:13.581Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 823 | 2024-02-15T00:06:13.582Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 824 | 2024-02-15T00:06:13.582Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 825 | 2024-02-15T00:06:13.615Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 826 | 2024-02-15T00:06:13.615Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 827 | 2024-02-15T00:06:13.615Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 828 | 2024-02-15T00:06:13.615Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 829 | 2024-02-15T00:06:13.615Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 830 | 2024-02-15T00:06:13.615Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 182876
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 437912f8-811f-42e6-a4ce-79f8c5aa9dae
response_code = 500
uri = /omicron-zones
|
| 831 | 2024-02-15T00:06:13.615Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "437912f8-811f-42e6-a4ce-79f8c5aa9dae", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:13 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "437912f8-811f-42e6-a4ce-79f8c5aa9dae" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 594.640615ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 832 | 2024-02-15T00:06:14.212Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 833 | 2024-02-15T00:06:14.213Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 834 | 2024-02-15T00:06:14.213Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 835 | 2024-02-15T00:06:14.233Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 836 | 2024-02-15T00:06:14.233Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 837 | 2024-02-15T00:06:14.251Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 838 | 2024-02-15T00:06:14.251Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 839 | 2024-02-15T00:06:14.270Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 840 | 2024-02-15T00:06:14.270Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 841 | 2024-02-15T00:06:14.289Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 842 | 2024-02-15T00:06:14.289Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 843 | 2024-02-15T00:06:14.307Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 844 | 2024-02-15T00:06:14.307Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 845 | 2024-02-15T00:06:14.325Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 846 | 2024-02-15T00:06:14.325Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 847 | 2024-02-15T00:06:14.344Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 848 | 2024-02-15T00:06:14.344Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 849 | 2024-02-15T00:06:14.362Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 850 | 2024-02-15T00:06:14.362Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 851 | 2024-02-15T00:06:14.362Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 852 | 2024-02-15T00:06:14.362Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 853 | 2024-02-15T00:06:14.395Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 854 | 2024-02-15T00:06:14.395Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 855 | 2024-02-15T00:06:14.395Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 856 | 2024-02-15T00:06:14.395Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 857 | 2024-02-15T00:06:14.395Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 858 | 2024-02-15T00:06:14.395Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 181878
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = ca536acb-ae50-4373-b072-67559f045dad
response_code = 500
uri = /omicron-zones
|
| 859 | 2024-02-15T00:06:14.395Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "ca536acb-ae50-4373-b072-67559f045dad", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "ca536acb-ae50-4373-b072-67559f045dad" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 891.620816ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 860 | 2024-02-15T00:06:15.290Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 861 | 2024-02-15T00:06:15.291Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 862 | 2024-02-15T00:06:15.291Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 863 | 2024-02-15T00:06:15.310Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 864 | 2024-02-15T00:06:15.310Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 865 | 2024-02-15T00:06:15.329Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 866 | 2024-02-15T00:06:15.329Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 867 | 2024-02-15T00:06:15.348Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 868 | 2024-02-15T00:06:15.348Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 869 | 2024-02-15T00:06:15.367Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 870 | 2024-02-15T00:06:15.367Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 871 | 2024-02-15T00:06:15.385Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 872 | 2024-02-15T00:06:15.385Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 873 | 2024-02-15T00:06:15.403Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 874 | 2024-02-15T00:06:15.403Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 875 | 2024-02-15T00:06:15.421Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 876 | 2024-02-15T00:06:15.421Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 877 | 2024-02-15T00:06:15.438Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 878 | 2024-02-15T00:06:15.439Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 879 | 2024-02-15T00:06:15.439Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 880 | 2024-02-15T00:06:15.439Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 881 | 2024-02-15T00:06:15.471Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 882 | 2024-02-15T00:06:15.471Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 883 | 2024-02-15T00:06:15.471Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 884 | 2024-02-15T00:06:15.471Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 885 | 2024-02-15T00:06:15.471Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 886 | 2024-02-15T00:06:15.471Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 180340
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 20edb31e-9670-4246-8f28-bc6a1d8793ec
response_code = 500
uri = /omicron-zones
|
| 887 | 2024-02-15T00:06:15.472Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "20edb31e-9670-4246-8f28-bc6a1d8793ec", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:15 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "20edb31e-9670-4246-8f28-bc6a1d8793ec" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 774.770631ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 888 | 2024-02-15T00:06:16.250Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 889 | 2024-02-15T00:06:16.251Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 890 | 2024-02-15T00:06:16.251Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 891 | 2024-02-15T00:06:16.271Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 892 | 2024-02-15T00:06:16.273Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 893 | 2024-02-15T00:06:16.288Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 894 | 2024-02-15T00:06:16.288Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 895 | 2024-02-15T00:06:16.306Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 896 | 2024-02-15T00:06:16.306Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 897 | 2024-02-15T00:06:16.325Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 898 | 2024-02-15T00:06:16.325Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 899 | 2024-02-15T00:06:16.342Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 900 | 2024-02-15T00:06:16.342Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 901 | 2024-02-15T00:06:16.360Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 902 | 2024-02-15T00:06:16.360Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 903 | 2024-02-15T00:06:16.377Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 904 | 2024-02-15T00:06:16.377Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 905 | 2024-02-15T00:06:16.395Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 906 | 2024-02-15T00:06:16.395Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 907 | 2024-02-15T00:06:16.396Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 908 | 2024-02-15T00:06:16.396Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 909 | 2024-02-15T00:06:16.427Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 910 | 2024-02-15T00:06:16.427Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 911 | 2024-02-15T00:06:16.427Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 912 | 2024-02-15T00:06:16.427Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 913 | 2024-02-15T00:06:16.428Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 914 | 2024-02-15T00:06:16.428Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 176549
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = bba73c52-4d75-4313-bef9-7e035c714d16
response_code = 500
uri = /omicron-zones
|
| 915 | 2024-02-15T00:06:16.428Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "bba73c52-4d75-4313-bef9-7e035c714d16", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:16 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "bba73c52-4d75-4313-bef9-7e035c714d16" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 908.329474ms
sled_address = [fd00:1122:3344:101::1]:12345
|
| 916 | 2024-02-15T00:06:17.339Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 917 | 2024-02-15T00:06:17.340Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 918 | 2024-02-15T00:06:17.340Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 919 | 2024-02-15T00:06:17.361Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 920 | 2024-02-15T00:06:17.361Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 921 | 2024-02-15T00:06:17.381Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 922 | 2024-02-15T00:06:17.381Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 923 | 2024-02-15T00:06:17.400Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 924 | 2024-02-15T00:06:17.400Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 925 | 2024-02-15T00:06:17.418Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 926 | 2024-02-15T00:06:17.418Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 927 | 2024-02-15T00:06:17.436Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 928 | 2024-02-15T00:06:17.436Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 929 | 2024-02-15T00:06:17.455Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 930 | 2024-02-15T00:06:17.455Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 931 | 2024-02-15T00:06:17.474Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 932 | 2024-02-15T00:06:17.474Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 933 | 2024-02-15T00:06:17.492Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 934 | 2024-02-15T00:06:17.492Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 935 | 2024-02-15T00:06:17.492Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 936 | 2024-02-15T00:06:17.492Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 937 | 2024-02-15T00:06:17.525Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 938 | 2024-02-15T00:06:17.525Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 939 | 2024-02-15T00:06:17.525Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 940 | 2024-02-15T00:06:17.525Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 941 | 2024-02-15T00:06:17.525Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 942 | 2024-02-15T00:06:17.525Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 184605
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = de2d49b1-619c-4d4a-ab92-25c19e876980
response_code = 500
uri = /omicron-zones
|
| 943 | 2024-02-15T00:06:17.525Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "de2d49b1-619c-4d4a-ab92-25c19e876980", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:17 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "de2d49b1-619c-4d4a-ab92-25c19e876980" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 1.467555042s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 944 | 2024-02-15T00:06:18.999Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 945 | 2024-02-15T00:06:18.999Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 946 | 2024-02-15T00:06:18.999Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 947 | 2024-02-15T00:06:19.019Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 948 | 2024-02-15T00:06:19.019Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 949 | 2024-02-15T00:06:19.039Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 950 | 2024-02-15T00:06:19.039Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 951 | 2024-02-15T00:06:19.057Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 952 | 2024-02-15T00:06:19.057Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 953 | 2024-02-15T00:06:19.074Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 954 | 2024-02-15T00:06:19.074Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 955 | 2024-02-15T00:06:19.092Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 956 | 2024-02-15T00:06:19.092Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 957 | 2024-02-15T00:06:19.110Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 958 | 2024-02-15T00:06:19.110Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 959 | 2024-02-15T00:06:19.128Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 960 | 2024-02-15T00:06:19.128Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 961 | 2024-02-15T00:06:19.147Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 962 | 2024-02-15T00:06:19.147Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 963 | 2024-02-15T00:06:19.147Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 964 | 2024-02-15T00:06:19.147Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 965 | 2024-02-15T00:06:19.179Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 966 | 2024-02-15T00:06:19.179Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 967 | 2024-02-15T00:06:19.179Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 968 | 2024-02-15T00:06:19.179Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 969 | 2024-02-15T00:06:19.179Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 970 | 2024-02-15T00:06:19.180Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 179897
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = acffbe91-ba51-4485-b039-77cdfbd929db
response_code = 500
uri = /omicron-zones
|
| 971 | 2024-02-15T00:06:19.180Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "acffbe91-ba51-4485-b039-77cdfbd929db", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:18 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "acffbe91-ba51-4485-b039-77cdfbd929db" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 1.052290343s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 972 | 2024-02-15T00:06:20.236Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 973 | 2024-02-15T00:06:20.237Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 974 | 2024-02-15T00:06:20.237Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 975 | 2024-02-15T00:06:20.257Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 976 | 2024-02-15T00:06:20.257Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 977 | 2024-02-15T00:06:20.275Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 978 | 2024-02-15T00:06:20.275Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 979 | 2024-02-15T00:06:20.293Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 980 | 2024-02-15T00:06:20.293Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 981 | 2024-02-15T00:06:20.311Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 982 | 2024-02-15T00:06:20.311Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 983 | 2024-02-15T00:06:20.328Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 984 | 2024-02-15T00:06:20.328Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 985 | 2024-02-15T00:06:20.347Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 986 | 2024-02-15T00:06:20.347Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 987 | 2024-02-15T00:06:20.365Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 988 | 2024-02-15T00:06:20.366Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 989 | 2024-02-15T00:06:20.383Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 990 | 2024-02-15T00:06:20.383Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 991 | 2024-02-15T00:06:20.384Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 992 | 2024-02-15T00:06:20.384Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 993 | 2024-02-15T00:06:20.416Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 994 | 2024-02-15T00:06:20.416Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 995 | 2024-02-15T00:06:20.416Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 996 | 2024-02-15T00:06:20.416Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 997 | 2024-02-15T00:06:20.416Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 998 | 2024-02-15T00:06:20.416Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 179335
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = f2023db3-d408-45ad-ad1e-ebcb370efcb6
response_code = 500
uri = /omicron-zones
|
| 999 | 2024-02-15T00:06:20.417Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f2023db3-d408-45ad-ad1e-ebcb370efcb6", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f2023db3-d408-45ad-ad1e-ebcb370efcb6" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 1.530985016s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1000 | 2024-02-15T00:06:20.619Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:37377
|
| 1001 | 2024-02-15T00:06:20.620Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 611
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:37377
req_id = 0155f83e-3d81-414c-aed7-f9584b1a1eb2
response_code = 200
uri = /baseboard
|
| 1002 | 2024-02-15T00:06:21.953Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1003 | 2024-02-15T00:06:21.953Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1004 | 2024-02-15T00:06:21.953Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1005 | 2024-02-15T00:06:21.974Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1006 | 2024-02-15T00:06:21.974Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1007 | 2024-02-15T00:06:21.992Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1008 | 2024-02-15T00:06:21.992Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1009 | 2024-02-15T00:06:22.010Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1010 | 2024-02-15T00:06:22.010Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1011 | 2024-02-15T00:06:22.028Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1012 | 2024-02-15T00:06:22.028Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1013 | 2024-02-15T00:06:22.047Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1014 | 2024-02-15T00:06:22.047Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1015 | 2024-02-15T00:06:22.066Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1016 | 2024-02-15T00:06:22.066Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1017 | 2024-02-15T00:06:22.085Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1018 | 2024-02-15T00:06:22.085Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1019 | 2024-02-15T00:06:22.101Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1020 | 2024-02-15T00:06:22.101Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1021 | 2024-02-15T00:06:22.102Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1022 | 2024-02-15T00:06:22.102Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1023 | 2024-02-15T00:06:22.134Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1024 | 2024-02-15T00:06:22.134Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1025 | 2024-02-15T00:06:22.134Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1026 | 2024-02-15T00:06:22.134Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1027 | 2024-02-15T00:06:22.134Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1028 | 2024-02-15T00:06:22.134Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 180426
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 3540f5de-0515-4c97-abd2-839d41c992c3
response_code = 500
uri = /omicron-zones
|
| 1029 | 2024-02-15T00:06:22.134Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3540f5de-0515-4c97-abd2-839d41c992c3", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:21 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3540f5de-0515-4c97-abd2-839d41c992c3" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 2.018835481s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1030 | 2024-02-15T00:06:24.160Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1031 | 2024-02-15T00:06:24.161Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1032 | 2024-02-15T00:06:24.161Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1033 | 2024-02-15T00:06:24.183Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1034 | 2024-02-15T00:06:24.184Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1035 | 2024-02-15T00:06:24.202Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1036 | 2024-02-15T00:06:24.202Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1037 | 2024-02-15T00:06:24.221Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1038 | 2024-02-15T00:06:24.221Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1039 | 2024-02-15T00:06:24.240Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1040 | 2024-02-15T00:06:24.240Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1041 | 2024-02-15T00:06:24.259Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1042 | 2024-02-15T00:06:24.259Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1043 | 2024-02-15T00:06:24.277Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1044 | 2024-02-15T00:06:24.277Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1045 | 2024-02-15T00:06:24.295Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1046 | 2024-02-15T00:06:24.295Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1047 | 2024-02-15T00:06:24.313Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1048 | 2024-02-15T00:06:24.313Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1049 | 2024-02-15T00:06:24.314Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1050 | 2024-02-15T00:06:24.314Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1051 | 2024-02-15T00:06:24.347Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1052 | 2024-02-15T00:06:24.347Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1053 | 2024-02-15T00:06:24.347Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1054 | 2024-02-15T00:06:24.347Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1055 | 2024-02-15T00:06:24.347Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1056 | 2024-02-15T00:06:24.348Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 186337
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 43f83293-b20f-45d5-8a08-18bee3f03214
response_code = 500
uri = /omicron-zones
|
| 1057 | 2024-02-15T00:06:24.348Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "43f83293-b20f-45d5-8a08-18bee3f03214", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:24 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "43f83293-b20f-45d5-8a08-18bee3f03214" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 1.674943892s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1058 | 2024-02-15T00:06:26.028Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1059 | 2024-02-15T00:06:26.028Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1060 | 2024-02-15T00:06:26.028Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1061 | 2024-02-15T00:06:26.049Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1062 | 2024-02-15T00:06:26.049Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1063 | 2024-02-15T00:06:26.067Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1064 | 2024-02-15T00:06:26.067Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1065 | 2024-02-15T00:06:26.085Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1066 | 2024-02-15T00:06:26.085Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1067 | 2024-02-15T00:06:26.104Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1068 | 2024-02-15T00:06:26.104Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1069 | 2024-02-15T00:06:26.122Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1070 | 2024-02-15T00:06:26.122Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1071 | 2024-02-15T00:06:26.139Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1072 | 2024-02-15T00:06:26.140Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1073 | 2024-02-15T00:06:26.158Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1074 | 2024-02-15T00:06:26.158Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1075 | 2024-02-15T00:06:26.175Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1076 | 2024-02-15T00:06:26.175Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1077 | 2024-02-15T00:06:26.175Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1078 | 2024-02-15T00:06:26.175Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1079 | 2024-02-15T00:06:26.208Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1080 | 2024-02-15T00:06:26.208Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1081 | 2024-02-15T00:06:26.208Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1082 | 2024-02-15T00:06:26.208Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1083 | 2024-02-15T00:06:26.208Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1084 | 2024-02-15T00:06:26.208Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 179079
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = f4d4aca0-04b8-4399-9400-e5f11adcf655
response_code = 500
uri = /omicron-zones
|
| 1085 | 2024-02-15T00:06:26.208Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f4d4aca0-04b8-4399-9400-e5f11adcf655", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:26 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f4d4aca0-04b8-4399-9400-e5f11adcf655" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 4.441972646s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1086 | 2024-02-15T00:06:30.664Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1087 | 2024-02-15T00:06:30.665Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1088 | 2024-02-15T00:06:30.665Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1089 | 2024-02-15T00:06:30.685Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1090 | 2024-02-15T00:06:30.685Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1091 | 2024-02-15T00:06:30.705Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1092 | 2024-02-15T00:06:30.705Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1093 | 2024-02-15T00:06:30.722Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1094 | 2024-02-15T00:06:30.722Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1095 | 2024-02-15T00:06:30.741Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1096 | 2024-02-15T00:06:30.741Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1097 | 2024-02-15T00:06:30.759Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1098 | 2024-02-15T00:06:30.759Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1099 | 2024-02-15T00:06:30.777Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1100 | 2024-02-15T00:06:30.777Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1101 | 2024-02-15T00:06:30.794Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1102 | 2024-02-15T00:06:30.795Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1103 | 2024-02-15T00:06:30.813Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1104 | 2024-02-15T00:06:30.813Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1105 | 2024-02-15T00:06:30.813Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1106 | 2024-02-15T00:06:30.813Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1107 | 2024-02-15T00:06:30.846Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1108 | 2024-02-15T00:06:30.846Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1109 | 2024-02-15T00:06:30.846Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1110 | 2024-02-15T00:06:30.846Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1111 | 2024-02-15T00:06:30.846Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1112 | 2024-02-15T00:06:30.846Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 180758
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 836c49c9-dd38-4739-806b-b1da78a7368c
response_code = 500
uri = /omicron-zones
|
| 1113 | 2024-02-15T00:06:30.846Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "836c49c9-dd38-4739-806b-b1da78a7368c", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:30 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "836c49c9-dd38-4739-806b-b1da78a7368c" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 2.913907403s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1114 | 2024-02-15T00:06:33.769Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1115 | 2024-02-15T00:06:33.769Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1116 | 2024-02-15T00:06:33.769Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1117 | 2024-02-15T00:06:33.791Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1118 | 2024-02-15T00:06:33.791Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1119 | 2024-02-15T00:06:33.809Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1120 | 2024-02-15T00:06:33.809Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1121 | 2024-02-15T00:06:33.828Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1122 | 2024-02-15T00:06:33.828Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1123 | 2024-02-15T00:06:33.846Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1124 | 2024-02-15T00:06:33.846Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1125 | 2024-02-15T00:06:33.865Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1126 | 2024-02-15T00:06:33.865Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1127 | 2024-02-15T00:06:33.883Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1128 | 2024-02-15T00:06:33.883Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1129 | 2024-02-15T00:06:33.901Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1130 | 2024-02-15T00:06:33.901Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1131 | 2024-02-15T00:06:33.919Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1132 | 2024-02-15T00:06:33.919Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1133 | 2024-02-15T00:06:33.919Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1134 | 2024-02-15T00:06:33.919Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1135 | 2024-02-15T00:06:33.951Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1136 | 2024-02-15T00:06:33.951Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1137 | 2024-02-15T00:06:33.951Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1138 | 2024-02-15T00:06:33.951Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1139 | 2024-02-15T00:06:33.951Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1140 | 2024-02-15T00:06:33.952Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 181796
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 842c74b0-7ffb-46ab-883c-31720e7fb4a3
response_code = 500
uri = /omicron-zones
|
| 1141 | 2024-02-15T00:06:33.952Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "842c74b0-7ffb-46ab-883c-31720e7fb4a3", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:33 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "842c74b0-7ffb-46ab-883c-31720e7fb4a3" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 4.508646585s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1142 | 2024-02-15T00:06:38.474Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1143 | 2024-02-15T00:06:38.475Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1144 | 2024-02-15T00:06:38.475Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1145 | 2024-02-15T00:06:38.496Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1146 | 2024-02-15T00:06:38.496Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1147 | 2024-02-15T00:06:38.514Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1148 | 2024-02-15T00:06:38.514Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1149 | 2024-02-15T00:06:38.532Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1150 | 2024-02-15T00:06:38.532Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1151 | 2024-02-15T00:06:38.552Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1152 | 2024-02-15T00:06:38.552Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1153 | 2024-02-15T00:06:38.571Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1154 | 2024-02-15T00:06:38.571Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1155 | 2024-02-15T00:06:38.588Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1156 | 2024-02-15T00:06:38.588Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1157 | 2024-02-15T00:06:38.606Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1158 | 2024-02-15T00:06:38.606Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1159 | 2024-02-15T00:06:38.624Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1160 | 2024-02-15T00:06:38.624Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1161 | 2024-02-15T00:06:38.624Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1162 | 2024-02-15T00:06:38.624Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1163 | 2024-02-15T00:06:38.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1164 | 2024-02-15T00:06:38.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1165 | 2024-02-15T00:06:38.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1166 | 2024-02-15T00:06:38.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1167 | 2024-02-15T00:06:38.656Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1168 | 2024-02-15T00:06:38.656Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 181210
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 57ba6f99-c3f6-4617-ba9e-70f6c8ef56dd
response_code = 500
uri = /omicron-zones
|
| 1169 | 2024-02-15T00:06:38.656Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "57ba6f99-c3f6-4617-ba9e-70f6c8ef56dd", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:38 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "57ba6f99-c3f6-4617-ba9e-70f6c8ef56dd" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 7.776098451s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1170 | 2024-02-15T00:06:46.445Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1171 | 2024-02-15T00:06:46.445Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1172 | 2024-02-15T00:06:46.445Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1173 | 2024-02-15T00:06:46.466Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1174 | 2024-02-15T00:06:46.466Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1175 | 2024-02-15T00:06:46.484Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1176 | 2024-02-15T00:06:46.484Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1177 | 2024-02-15T00:06:46.502Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1178 | 2024-02-15T00:06:46.502Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1179 | 2024-02-15T00:06:46.520Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1180 | 2024-02-15T00:06:46.520Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1181 | 2024-02-15T00:06:46.538Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1182 | 2024-02-15T00:06:46.539Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1183 | 2024-02-15T00:06:46.557Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1184 | 2024-02-15T00:06:46.557Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1185 | 2024-02-15T00:06:46.575Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1186 | 2024-02-15T00:06:46.575Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1187 | 2024-02-15T00:06:46.593Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1188 | 2024-02-15T00:06:46.594Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1189 | 2024-02-15T00:06:46.594Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1190 | 2024-02-15T00:06:46.594Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1191 | 2024-02-15T00:06:46.626Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1192 | 2024-02-15T00:06:46.626Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1193 | 2024-02-15T00:06:46.626Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1194 | 2024-02-15T00:06:46.626Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1195 | 2024-02-15T00:06:46.626Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1196 | 2024-02-15T00:06:46.626Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 181425
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = e4791c08-4804-40b0-b7ee-b20ab58eb73b
response_code = 500
uri = /omicron-zones
|
| 1197 | 2024-02-15T00:06:46.627Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e4791c08-4804-40b0-b7ee-b20ab58eb73b", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:46 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e4791c08-4804-40b0-b7ee-b20ab58eb73b" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 3.995345707s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1198 | 2024-02-15T00:06:50.623Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1199 | 2024-02-15T00:06:50.624Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1200 | 2024-02-15T00:06:50.624Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1201 | 2024-02-15T00:06:50.645Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1202 | 2024-02-15T00:06:50.645Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1203 | 2024-02-15T00:06:50.663Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1204 | 2024-02-15T00:06:50.663Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1205 | 2024-02-15T00:06:50.681Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1206 | 2024-02-15T00:06:50.681Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1207 | 2024-02-15T00:06:50.700Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1208 | 2024-02-15T00:06:50.700Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1209 | 2024-02-15T00:06:50.713Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60177
|
| 1210 | 2024-02-15T00:06:50.713Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 151
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60177
req_id = 47ee68e7-70af-4a66-b2bd-8dbef797b4b5
response_code = 200
uri = /baseboard
|
| 1211 | 2024-02-15T00:06:50.720Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1212 | 2024-02-15T00:06:50.720Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1213 | 2024-02-15T00:06:50.740Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1214 | 2024-02-15T00:06:50.740Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1215 | 2024-02-15T00:06:50.758Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1216 | 2024-02-15T00:06:50.758Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1217 | 2024-02-15T00:06:50.776Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1218 | 2024-02-15T00:06:50.776Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1219 | 2024-02-15T00:06:50.776Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1220 | 2024-02-15T00:06:50.776Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1221 | 2024-02-15T00:06:50.808Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1222 | 2024-02-15T00:06:50.808Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1223 | 2024-02-15T00:06:50.808Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1224 | 2024-02-15T00:06:50.808Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1225 | 2024-02-15T00:06:50.808Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1226 | 2024-02-15T00:06:50.809Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 185449
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 5dde27f6-c8b4-4682-b236-237c6b5a3548
response_code = 500
uri = /omicron-zones
|
| 1227 | 2024-02-15T00:06:50.809Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "5dde27f6-c8b4-4682-b236-237c6b5a3548", "content-length": "124", "date": "Thu, 15 Feb 2024 00:06:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "5dde27f6-c8b4-4682-b236-237c6b5a3548" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 10.193353963s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1228 | 2024-02-15T00:07:01.003Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1229 | 2024-02-15T00:07:01.004Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1230 | 2024-02-15T00:07:01.004Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1231 | 2024-02-15T00:07:01.026Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1232 | 2024-02-15T00:07:01.026Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1233 | 2024-02-15T00:07:01.046Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1234 | 2024-02-15T00:07:01.046Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1235 | 2024-02-15T00:07:01.064Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1236 | 2024-02-15T00:07:01.064Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1237 | 2024-02-15T00:07:01.083Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1238 | 2024-02-15T00:07:01.083Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1239 | 2024-02-15T00:07:01.101Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1240 | 2024-02-15T00:07:01.101Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1241 | 2024-02-15T00:07:01.119Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1242 | 2024-02-15T00:07:01.119Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1243 | 2024-02-15T00:07:01.138Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1244 | 2024-02-15T00:07:01.138Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1245 | 2024-02-15T00:07:01.156Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1246 | 2024-02-15T00:07:01.156Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1247 | 2024-02-15T00:07:01.156Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1248 | 2024-02-15T00:07:01.157Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1249 | 2024-02-15T00:07:01.189Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1250 | 2024-02-15T00:07:01.189Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1251 | 2024-02-15T00:07:01.189Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1252 | 2024-02-15T00:07:01.189Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1253 | 2024-02-15T00:07:01.189Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1254 | 2024-02-15T00:07:01.189Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 185106
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 75e30880-83e5-4385-9e4e-58d44528bdaf
response_code = 500
uri = /omicron-zones
|
| 1255 | 2024-02-15T00:07:01.189Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "75e30880-83e5-4385-9e4e-58d44528bdaf", "content-length": "124", "date": "Thu, 15 Feb 2024 00:07:01 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "75e30880-83e5-4385-9e4e-58d44528bdaf" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 10.043966205s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1256 | 2024-02-15T00:07:11.234Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1257 | 2024-02-15T00:07:11.235Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1258 | 2024-02-15T00:07:11.235Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1259 | 2024-02-15T00:07:11.256Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1260 | 2024-02-15T00:07:11.256Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1261 | 2024-02-15T00:07:11.275Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1262 | 2024-02-15T00:07:11.275Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1263 | 2024-02-15T00:07:11.293Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1264 | 2024-02-15T00:07:11.293Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1265 | 2024-02-15T00:07:11.311Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1266 | 2024-02-15T00:07:11.311Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1267 | 2024-02-15T00:07:11.329Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1268 | 2024-02-15T00:07:11.329Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1269 | 2024-02-15T00:07:11.347Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1270 | 2024-02-15T00:07:11.347Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1271 | 2024-02-15T00:07:11.366Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1272 | 2024-02-15T00:07:11.366Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1273 | 2024-02-15T00:07:11.384Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1274 | 2024-02-15T00:07:11.384Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1275 | 2024-02-15T00:07:11.384Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1276 | 2024-02-15T00:07:11.384Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1277 | 2024-02-15T00:07:11.417Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1278 | 2024-02-15T00:07:11.417Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1279 | 2024-02-15T00:07:11.417Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1280 | 2024-02-15T00:07:11.417Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1281 | 2024-02-15T00:07:11.417Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1282 | 2024-02-15T00:07:11.417Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 182863
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = 8930c7c4-ca7d-4a86-b623-7582efd1db5c
response_code = 500
uri = /omicron-zones
|
| 1283 | 2024-02-15T00:07:11.418Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8930c7c4-ca7d-4a86-b623-7582efd1db5c", "content-length": "124", "date": "Thu, 15 Feb 2024 00:07:11 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8930c7c4-ca7d-4a86-b623-7582efd1db5c" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 15.43107045s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1284 | 2024-02-15T00:07:20.745Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:38399
|
| 1285 | 2024-02-15T00:07:20.745Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 65
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:38399
req_id = 2f4cd75f-dfdf-4aee-9e6c-f3e67ec2d302
response_code = 200
uri = /baseboard
|
| 1286 | 2024-02-15T00:07:26.850Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1287 | 2024-02-15T00:07:26.850Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1288 | 2024-02-15T00:07:26.850Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1289 | 2024-02-15T00:07:26.872Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1290 | 2024-02-15T00:07:26.872Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1291 | 2024-02-15T00:07:26.890Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1292 | 2024-02-15T00:07:26.890Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1293 | 2024-02-15T00:07:26.909Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1294 | 2024-02-15T00:07:26.909Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1295 | 2024-02-15T00:07:26.928Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1296 | 2024-02-15T00:07:26.928Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1297 | 2024-02-15T00:07:26.945Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1298 | 2024-02-15T00:07:26.945Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1299 | 2024-02-15T00:07:26.963Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1300 | 2024-02-15T00:07:26.963Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1301 | 2024-02-15T00:07:26.981Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1302 | 2024-02-15T00:07:26.981Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1303 | 2024-02-15T00:07:27.000Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1304 | 2024-02-15T00:07:27.000Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1305 | 2024-02-15T00:07:27.000Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1306 | 2024-02-15T00:07:27.000Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1307 | 2024-02-15T00:07:27.032Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1308 | 2024-02-15T00:07:27.032Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1309 | 2024-02-15T00:07:27.032Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1310 | 2024-02-15T00:07:27.032Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1311 | 2024-02-15T00:07:27.032Z | WARN | SledAgent (ServiceManager): Zone failed to start
file = sled-agent/src/services.rs:2770
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1312 | 2024-02-15T00:07:27.033Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to initialize zones: [("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0", TimeNotSynchronized), ("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e", TimeNotSynchronized), ("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e", TimeNotSynchronized), ("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d", TimeNotSynchronized), ("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b", TimeNotSynchronized)]
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837
latency_us = 182602
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = c2dc2c0c-a09a-49a6-b9d2-ecf7d3afb7e7
response_code = 500
uri = /omicron-zones
|
| 1313 | 2024-02-15T00:07:27.033Z | WARN | SledAgent (RSS): failed to initialize Omicron zones
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c2dc2c0c-a09a-49a6-b9d2-ecf7d3afb7e7", "content-length": "124", "date": "Thu, 15 Feb 2024 00:07:26 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c2dc2c0c-a09a-49a6-b9d2-ecf7d3afb7e7" }
file = sled-agent/src/rack_setup/service.rs:337
retry_after = 16.492442452s
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1314 | 2024-02-15T00:07:43.526Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1315 | 2024-02-15T00:07:43.527Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1316 | 2024-02-15T00:07:43.527Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1317 | 2024-02-15T00:07:43.548Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1318 | 2024-02-15T00:07:43.548Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1319 | 2024-02-15T00:07:43.568Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1320 | 2024-02-15T00:07:43.568Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1321 | 2024-02-15T00:07:43.587Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1322 | 2024-02-15T00:07:43.587Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1323 | 2024-02-15T00:07:43.605Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1324 | 2024-02-15T00:07:43.605Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1325 | 2024-02-15T00:07:43.623Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1326 | 2024-02-15T00:07:43.623Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1327 | 2024-02-15T00:07:43.641Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1328 | 2024-02-15T00:07:43.641Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1329 | 2024-02-15T00:07:43.659Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1330 | 2024-02-15T00:07:43.659Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1331 | 2024-02-15T00:07:43.677Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1332 | 2024-02-15T00:07:43.677Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1333 | 2024-02-15T00:07:43.678Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1334 | 2024-02-15T00:07:43.678Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1335 | 2024-02-15T00:07:43.761Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1336 | 2024-02-15T00:07:43.761Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1337 | 2024-02-15T00:07:43.761Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1338 | 2024-02-15T00:07:43.761Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1339 | 2024-02-15T00:07:43.761Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1340 | 2024-02-15T00:07:43.854Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
file = illumos-utils/src/zone.rs:315
|
| 1341 | 2024-02-15T00:07:43.858Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
file = illumos-utils/src/zone.rs:315
|
| 1342 | 2024-02-15T00:07:43.862Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
file = illumos-utils/src/zone.rs:315
|
| 1343 | 2024-02-15T00:07:43.866Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
file = illumos-utils/src/zone.rs:315
|
| 1344 | 2024-02-15T00:07:43.871Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
file = illumos-utils/src/zone.rs:315
|
| 1345 | 2024-02-15T00:07:43.875Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
file = illumos-utils/src/zone.rs:354
|
| 1346 | 2024-02-15T00:07:43.880Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
file = illumos-utils/src/zone.rs:354
|
| 1347 | 2024-02-15T00:07:43.886Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
file = illumos-utils/src/zone.rs:354
|
| 1348 | 2024-02-15T00:07:43.890Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
file = illumos-utils/src/zone.rs:354
|
| 1349 | 2024-02-15T00:07:43.894Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
file = illumos-utils/src/zone.rs:354
|
| 1350 | 2024-02-15T00:07:47.270Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService5"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::3"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::3"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1351 | 2024-02-15T00:07:47.271Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1352 | 2024-02-15T00:07:47.358Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService9"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::7"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::7"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1353 | 2024-02-15T00:07:47.358Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1354 | 2024-02-15T00:07:47.409Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 44.616981ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1355 | 2024-02-15T00:07:47.463Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 75.854866ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1356 | 2024-02-15T00:07:47.577Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 33.015312ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1357 | 2024-02-15T00:07:47.592Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 119.995282ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1358 | 2024-02-15T00:07:47.600Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService7"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::6"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1359 | 2024-02-15T00:07:47.600Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1360 | 2024-02-15T00:07:47.635Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 109.627679ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1361 | 2024-02-15T00:07:47.647Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService6"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::5"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::5"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1362 | 2024-02-15T00:07:47.649Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1363 | 2024-02-15T00:07:47.804Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 145.944336ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1364 | 2024-02-15T00:07:47.840Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 248.017387ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1365 | 2024-02-15T00:07:47.851Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService8"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::4"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::4"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1366 | 2024-02-15T00:07:47.861Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1367 | 2024-02-15T00:07:48.014Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 334.608036ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1368 | 2024-02-15T00:07:48.100Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 51.090608ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1369 | 2024-02-15T00:07:48.123Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 440.576443ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1370 | 2024-02-15T00:07:48.175Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 126.801922ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1371 | 2024-02-15T00:07:48.373Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 125.456483ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1372 | 2024-02-15T00:07:48.439Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 61.356786ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1373 | 2024-02-15T00:07:48.494Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 838.481772ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1374 | 2024-02-15T00:07:48.573Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 64.617116ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1375 | 2024-02-15T00:07:48.639Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 1.333236879s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1376 | 2024-02-15T00:07:48.667Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 65.750781ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1377 | 2024-02-15T00:07:48.709Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 464.731426ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1378 | 2024-02-15T00:07:48.771Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 58.046938ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1379 | 2024-02-15T00:07:48.817Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 136.356603ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1380 | 2024-02-15T00:07:48.862Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 259.365779ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1381 | 2024-02-15T00:07:48.988Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 364.756567ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1382 | 2024-02-15T00:07:49.139Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 235.107295ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1383 | 2024-02-15T00:07:49.190Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 905.009034ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1384 | 2024-02-15T00:07:49.349Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 1.374427352s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1385 | 2024-02-15T00:07:49.376Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 887.361317ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1386 | 2024-02-15T00:07:49.394Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 737.434171ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1387 | 2024-02-15T00:07:49.999Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 610.804903ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1388 | 2024-02-15T00:07:50.107Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 1.178315673s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1389 | 2024-02-15T00:07:50.143Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 1.398100062s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1390 | 2024-02-15T00:07:50.276Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 830.811983ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1391 | 2024-02-15T00:07:50.618Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 1.049417893s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1392 | 2024-02-15T00:07:50.732Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 1.126838966s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1393 | 2024-02-15T00:07:50.776Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:58033
|
| 1394 | 2024-02-15T00:07:50.777Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 590
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:58033
req_id = bfd19ee9-52b1-4d31-b223-ff8bdb14f8ad
response_code = 200
uri = /baseboard
|
| 1395 | 2024-02-15T00:07:51.115Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 588.401952ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1396 | 2024-02-15T00:07:51.295Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 1.01760747s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1397 | 2024-02-15T00:07:51.553Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 1.223696881s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1398 | 2024-02-15T00:07:51.680Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 1.472569534s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1399 | 2024-02-15T00:07:51.716Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 734.87956ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1400 | 2024-02-15T00:07:51.873Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 638.314189ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1401 | 2024-02-15T00:07:52.322Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 688.150891ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1402 | 2024-02-15T00:07:52.460Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 812.136552ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1403 | 2024-02-15T00:07:52.520Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 1.155406172s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1404 | 2024-02-15T00:07:52.785Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 992.937964ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1405 | 2024-02-15T00:07:53.018Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 919.563977ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1406 | 2024-02-15T00:07:53.160Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 839.15547ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1407 | 2024-02-15T00:07:53.280Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 730.264069ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1408 | 2024-02-15T00:07:53.685Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 1.073708384s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1409 | 2024-02-15T00:07:53.787Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 891.020902ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1410 | 2024-02-15T00:07:53.946Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 505.875474ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1411 | 2024-02-15T00:07:54.011Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 738.241885ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1412 | 2024-02-15T00:07:54.020Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 1.466933899s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1413 | 2024-02-15T00:07:54.463Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 1.399297859s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1414 | 2024-02-15T00:07:54.687Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 996.054405ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1415 | 2024-02-15T00:07:54.758Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e") failed: Property not found. retry in 1.038393089s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1416 | 2024-02-15T00:07:54.768Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 983.932403ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1417 | 2024-02-15T00:07:55.498Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 1.119965972s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1418 | 2024-02-15T00:07:55.692Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 898.473788ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1419 | 2024-02-15T00:07:55.761Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 1.003766211s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1420 | 2024-02-15T00:07:55.871Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 867.172141ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1421 | 2024-02-15T00:07:56.600Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 587.938604ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1422 | 2024-02-15T00:07:56.626Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 1.385184307s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1423 | 2024-02-15T00:07:56.748Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 1.007988853s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1424 | 2024-02-15T00:07:56.773Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 577.466629ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1425 | 2024-02-15T00:07:57.204Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 1.258406012s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1426 | 2024-02-15T00:07:57.363Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 854.689973ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1427 | 2024-02-15T00:07:57.766Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 932.347224ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1428 | 2024-02-15T00:07:58.021Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 1.300694016s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1429 | 2024-02-15T00:07:58.227Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 638.176911ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1430 | 2024-02-15T00:07:58.472Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 672.093438ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1431 | 2024-02-15T00:07:58.710Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 1.36979985s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1432 | 2024-02-15T00:07:58.875Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 1.245272247s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1433 | 2024-02-15T00:07:59.154Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 1.338248262s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1434 | 2024-02-15T00:07:59.331Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 857.146472ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1435 | 2024-02-15T00:08:00.089Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 1.333455842s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1436 | 2024-02-15T00:08:00.129Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 832.381786ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1437 | 2024-02-15T00:08:00.197Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 1.205472086s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1438 | 2024-02-15T00:08:00.500Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 1.250029271s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1439 | 2024-02-15T00:08:00.970Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 1.408693261s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1440 | 2024-02-15T00:08:01.412Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 721.024447ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1441 | 2024-02-15T00:08:01.431Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 943.022815ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1442 | 2024-02-15T00:08:01.761Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 1.310614611s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1443 | 2024-02-15T00:08:02.142Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 1.166113915s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1444 | 2024-02-15T00:08:02.383Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 973.826299ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1445 | 2024-02-15T00:08:02.391Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 974.039846ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1446 | 2024-02-15T00:08:03.079Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 1.394816146s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1447 | 2024-02-15T00:08:03.316Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 770.925159ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1448 | 2024-02-15T00:08:03.366Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 1.46536834s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1449 | 2024-02-15T00:08:03.373Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 1.419437928s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1450 | 2024-02-15T00:08:04.095Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e") failed: Property not found. retry in 1.041590593s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1451 | 2024-02-15T00:08:04.483Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b") failed: Property not found. retry in 611.546718ms
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1452 | 2024-02-15T00:08:04.802Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0") failed: Property not found. retry in 1.142208723s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1453 | 2024-02-15T00:08:04.839Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d") failed: Property not found. retry in 1.206018926s
file = illumos-utils/src/svc.rs:36
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1454 | 2024-02-15T00:08:06.071Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_cockroachdb_b157dbbc-477a-46c1-937a-6d9e521e4b6e
|
| 1455 | 2024-02-15T00:08:06.071Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_cockroachdb_21b3ce97-9e15-423d-932c-6a0f94f9763e
|
| 1456 | 2024-02-15T00:08:06.072Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_cockroachdb_f1f5bea5-c368-4f8f-82c1-780dc85b869d
|
| 1457 | 2024-02-15T00:08:06.072Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_cockroachdb_b70f6ed6-97c6-48eb-aefa-25fab0f23c6b
|
| 1458 | 2024-02-15T00:08:06.072Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_cockroachdb_527b0aec-e82d-4a9d-bb6e-b3e5df1899a0
|
| 1459 | 2024-02-15T00:08:06.072Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:198
|
| 1460 | 2024-02-15T00:08:06.072Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:198
|
| 1461 | 2024-02-15T00:08:06.072Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 22547458
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:61545
req_id = eee17f95-0eeb-4dcd-88be-cea1cb11887b
response_code = 204
uri = /omicron-zones
|
| 1462 | 2024-02-15T00:08:06.106Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:49560
|
| 1463 | 2024-02-15T00:08:06.139Z | INFO | SledAgent (ServiceManager): Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221
file = sled-agent/src/services.rs:3165
|
| 1464 | 2024-02-15T00:08:16.310Z | INFO | SledAgent (ServiceManager): Formatting CRDB
file = sled-agent/src/services.rs:3183
|
| 1465 | 2024-02-15T00:08:20.817Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64504
|
| 1466 | 2024-02-15T00:08:20.818Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 374
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64504
req_id = 41be91b1-f309-4d66-910f-2df9acf54d3b
response_code = 200
uri = /baseboard
|
| 1467 | 2024-02-15T00:08:27.212Z | INFO | SledAgent (ServiceManager): Formatting CRDB - Completed
file = sled-agent/src/services.rs:3206
|
| 1468 | 2024-02-15T00:08:27.213Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 21107421
local_addr = [fd00:1122:3344:101::1]:12345
method = POST
remote_addr = [fd00:1122:3344:101::1]:49560
req_id = ac90b76c-777e-4432-b042-76c3e7faf2bf
response_code = 204
uri = /cockroachdb
|
| 1469 | 2024-02-15T00:08:27.255Z | INFO | SledAgent (RSS): attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: eaa4434d-a6bd-48f6-beb2-151404cd7796, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a3944496-e534-42b6-afe3-78830d746249, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: ["0.pool.ntp.org"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 39552c48-8250-42cd-9881-89d1ecad7193, kind: Service { id: 06e1927a-c14e-4e46-8691-4b4ad6c57bc7 }, name: Name("ntp-06e1927a-c14e-4e46-8691-4b4ad6c57bc7"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 124])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1999ec12-789c-40c8-83be-a68bad0fad69, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 052b5be3-98fd-4823-9614-96d5055e333c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a0570de6-fbf4-4073-9e84-a6e01264b6a9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: b2eaae1a-392a-411d-b8cc-0dff26ddef0c, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9324d491-c0d2-436b-aafc-efea5edebe21, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 7d74b4bc-c408-4088-9473-a933f16aef97, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ad7ed868-02d0-452d-8c54-23c7ef814af4, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c22ce43b-b4c5-453b-9120-505102876329, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 1b32292f-6190-435a-87d8-7fdbe1a65f21, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 661755cb-0b27-4b29-a851-a3a45274279d, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 83fba088-495e-4423-abe6-b30a70233d4c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: c7373a32-5911-4b61-b967-b9351fe9441e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 475bc82c-0c96-4f51-8fdf-e66178c8f3fa, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cad70807-3779-4dbd-b5e8-a3a0e7563ca7, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b1234f77-5c34-4c96-bce0-c457facfb566, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3773d36c-8002-400f-9f73-719a40bd0f35, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }
file = sled-agent/src/rack_setup/service.rs:299
sled_address = [fd00:1122:3344:101::1]:12345
|
| 1470 | 2024-02-15T00:08:27.256Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:56765
|
| 1471 | 2024-02-15T00:08:27.257Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1472 | 2024-02-15T00:08:27.257Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: eaa4434d-a6bd-48f6-beb2-151404cd7796, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1473 | 2024-02-15T00:08:27.280Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1474 | 2024-02-15T00:08:27.280Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: a3944496-e534-42b6-afe3-78830d746249, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1475 | 2024-02-15T00:08:27.300Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1476 | 2024-02-15T00:08:27.300Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 5e4f48e8-b987-45c0-8c5b-523ef542f945, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1477 | 2024-02-15T00:08:27.319Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1478 | 2024-02-15T00:08:27.319Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b157dbbc-477a-46c1-937a-6d9e521e4b6e, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1479 | 2024-02-15T00:08:27.338Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1480 | 2024-02-15T00:08:27.338Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b70f6ed6-97c6-48eb-aefa-25fab0f23c6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1481 | 2024-02-15T00:08:27.356Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1482 | 2024-02-15T00:08:27.356Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 21b3ce97-9e15-423d-932c-6a0f94f9763e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1483 | 2024-02-15T00:08:27.374Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1484 | 2024-02-15T00:08:27.374Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: f1f5bea5-c368-4f8f-82c1-780dc85b869d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1485 | 2024-02-15T00:08:27.392Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1486 | 2024-02-15T00:08:27.392Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 527b0aec-e82d-4a9d-bb6e-b3e5df1899a0, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1487 | 2024-02-15T00:08:27.410Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1488 | 2024-02-15T00:08:27.410Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1489 | 2024-02-15T00:08:27.564Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1999ec12-789c-40c8-83be-a68bad0fad69, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1490 | 2024-02-15T00:08:27.564Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 1999ec12-789c-40c8-83be-a68bad0fad69, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1491 | 2024-02-15T00:08:27.691Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b2eaae1a-392a-411d-b8cc-0dff26ddef0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1492 | 2024-02-15T00:08:27.691Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b2eaae1a-392a-411d-b8cc-0dff26ddef0c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1493 | 2024-02-15T00:08:27.810Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c22ce43b-b4c5-453b-9120-505102876329, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1494 | 2024-02-15T00:08:27.810Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: c22ce43b-b4c5-453b-9120-505102876329, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1495 | 2024-02-15T00:08:27.937Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 1b32292f-6190-435a-87d8-7fdbe1a65f21, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1496 | 2024-02-15T00:08:27.937Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 1b32292f-6190-435a-87d8-7fdbe1a65f21, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1497 | 2024-02-15T00:08:28.054Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 661755cb-0b27-4b29-a851-a3a45274279d, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1498 | 2024-02-15T00:08:28.054Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 661755cb-0b27-4b29-a851-a3a45274279d, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1499 | 2024-02-15T00:08:28.178Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 83fba088-495e-4423-abe6-b30a70233d4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1500 | 2024-02-15T00:08:28.178Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 83fba088-495e-4423-abe6-b30a70233d4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1501 | 2024-02-15T00:08:28.293Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: c7373a32-5911-4b61-b967-b9351fe9441e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1502 | 2024-02-15T00:08:28.293Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: c7373a32-5911-4b61-b967-b9351fe9441e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1503 | 2024-02-15T00:08:28.421Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 475bc82c-0c96-4f51-8fdf-e66178c8f3fa, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1504 | 2024-02-15T00:08:28.422Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 475bc82c-0c96-4f51-8fdf-e66178c8f3fa, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1505 | 2024-02-15T00:08:28.563Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: cad70807-3779-4dbd-b5e8-a3a0e7563ca7, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1506 | 2024-02-15T00:08:28.563Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: cad70807-3779-4dbd-b5e8-a3a0e7563ca7, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1507 | 2024-02-15T00:08:28.680Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: b1234f77-5c34-4c96-bce0-c457facfb566, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1508 | 2024-02-15T00:08:28.680Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b1234f77-5c34-4c96-bce0-c457facfb566, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1509 | 2024-02-15T00:08:28.797Z | INFO | SledAgent (StorageManager): Received NewFilesystem(NewFilesystemRequest { dataset_id: 3773d36c-8002-400f-9f73-719a40bd0f35, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })
file = sled-storage/src/manager.rs:346
|
| 1510 | 2024-02-15T00:08:28.797Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 3773d36c-8002-400f-9f73-719a40bd0f35, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
file = sled-storage/src/manager.rs:565
|
| 1511 | 2024-02-15T00:08:28.889Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1512 | 2024-02-15T00:08:28.889Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1513 | 2024-02-15T00:08:28.890Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 1514 | 2024-02-15T00:08:28.890Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1515 | 2024-02-15T00:08:29.015Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:254
|
| 1516 | 2024-02-15T00:08:29.087Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:254
|
| 1517 | 2024-02-15T00:08:29.087Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1518 | 2024-02-15T00:08:29.087Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1519 | 2024-02-15T00:08:29.087Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1520 | 2024-02-15T00:08:29.087Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1521 | 2024-02-15T00:08:29.087Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1522 | 2024-02-15T00:08:29.087Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1523 | 2024-02-15T00:08:29.089Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:254
|
| 1524 | 2024-02-15T00:08:29.101Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1525 | 2024-02-15T00:08:29.278Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:254
|
| 1526 | 2024-02-15T00:08:29.278Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1527 | 2024-02-15T00:08:29.278Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1528 | 2024-02-15T00:08:29.278Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1529 | 2024-02-15T00:08:29.278Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1530 | 2024-02-15T00:08:29.278Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1531 | 2024-02-15T00:08:29.280Z | INFO | SledAgent (ServiceManager): Resolving switch zone addresses in DNS
file = sled-agent/src/bootstrap/early_networking.rs:254
|
| 1532 | 2024-02-15T00:08:29.292Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1533 | 2024-02-15T00:08:29.292Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1534 | 2024-02-15T00:08:29.293Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
file = illumos-utils/src/zone.rs:315
|
| 1535 | 2024-02-15T00:08:29.298Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
file = illumos-utils/src/zone.rs:315
|
| 1536 | 2024-02-15T00:08:29.304Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
file = illumos-utils/src/zone.rs:315
|
| 1537 | 2024-02-15T00:08:29.309Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
file = illumos-utils/src/zone.rs:315
|
| 1538 | 2024-02-15T00:08:29.314Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
file = illumos-utils/src/zone.rs:315
|
| 1539 | 2024-02-15T00:08:29.319Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
file = illumos-utils/src/zone.rs:315
|
| 1540 | 2024-02-15T00:08:29.324Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
file = illumos-utils/src/zone.rs:315
|
| 1541 | 2024-02-15T00:08:29.467Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
file = illumos-utils/src/zone.rs:354
|
| 1542 | 2024-02-15T00:08:29.471Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
file = illumos-utils/src/zone.rs:354
|
| 1543 | 2024-02-15T00:08:29.475Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
file = illumos-utils/src/zone.rs:354
|
| 1544 | 2024-02-15T00:08:29.480Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
file = illumos-utils/src/zone.rs:354
|
| 1545 | 2024-02-15T00:08:29.484Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
file = illumos-utils/src/zone.rs:354
|
| 1546 | 2024-02-15T00:08:29.489Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
file = illumos-utils/src/zone.rs:354
|
| 1547 | 2024-02-15T00:08:29.495Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
file = illumos-utils/src/zone.rs:354
|
| 1548 | 2024-02-15T00:08:29.500Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
file = illumos-utils/src/zone.rs:315
|
| 1549 | 2024-02-15T00:08:29.505Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
file = illumos-utils/src/zone.rs:315
|
| 1550 | 2024-02-15T00:08:29.511Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
file = illumos-utils/src/zone.rs:315
|
| 1551 | 2024-02-15T00:08:29.516Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
file = illumos-utils/src/zone.rs:315
|
| 1552 | 2024-02-15T00:08:29.521Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
file = illumos-utils/src/zone.rs:315
|
| 1553 | 2024-02-15T00:08:29.528Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
file = illumos-utils/src/zone.rs:315
|
| 1554 | 2024-02-15T00:08:29.533Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
file = illumos-utils/src/zone.rs:315
|
| 1555 | 2024-02-15T00:08:29.540Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
file = illumos-utils/src/zone.rs:354
|
| 1556 | 2024-02-15T00:08:29.546Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
file = illumos-utils/src/zone.rs:354
|
| 1557 | 2024-02-15T00:08:29.552Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
file = illumos-utils/src/zone.rs:354
|
| 1558 | 2024-02-15T00:08:29.557Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
file = illumos-utils/src/zone.rs:354
|
| 1559 | 2024-02-15T00:08:29.564Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
file = illumos-utils/src/zone.rs:354
|
| 1560 | 2024-02-15T00:08:29.569Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
file = illumos-utils/src/zone.rs:354
|
| 1561 | 2024-02-15T00:08:29.573Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
file = illumos-utils/src/zone.rs:354
|
| 1562 | 2024-02-15T00:08:29.614Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:271
|
| 1563 | 2024-02-15T00:08:29.653Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:271
|
| 1564 | 2024-02-15T00:08:29.696Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:271
|
| 1565 | 2024-02-15T00:08:29.737Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:271
|
| 1566 | 2024-02-15T00:08:29.775Z | INFO | SledAgent (ServiceManager): Querying MGS to determine switch location
addr = fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:271
|
| 1567 | 2024-02-15T00:08:29.777Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:296
|
| 1568 | 2024-02-15T00:08:29.777Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:188
|
| 1569 | 2024-02-15T00:08:29.836Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte1", _ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 201, 33]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte1" } }
|
| 1570 | 2024-02-15T00:08:29.837Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1268
zone_type = nexus
|
| 1571 | 2024-02-15T00:08:29.839Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:296
|
| 1572 | 2024-02-15T00:08:29.839Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:188
|
| 1573 | 2024-02-15T00:08:29.895Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte2", _ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 245, 172]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte2" } }
|
| 1574 | 2024-02-15T00:08:29.895Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1268
zone_type = external_dns
|
| 1575 | 2024-02-15T00:08:29.899Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:296
|
| 1576 | 2024-02-15T00:08:29.899Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:188
|
| 1577 | 2024-02-15T00:08:29.965Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte3", _ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 134, 173]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte3" } }
|
| 1578 | 2024-02-15T00:08:29.965Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1268
zone_type = nexus
|
| 1579 | 2024-02-15T00:08:29.967Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:296
|
| 1580 | 2024-02-15T00:08:29.967Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:188
|
| 1581 | 2024-02-15T00:08:30.027Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte4", _ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 161, 144]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte4" } }
|
| 1582 | 2024-02-15T00:08:30.028Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1268
zone_type = nexus
|
| 1583 | 2024-02-15T00:08:30.031Z | INFO | SledAgent (ServiceManager): Found Switch0 at fd00:1122:3344:101::2
file = sled-agent/src/bootstrap/early_networking.rs:296
|
| 1584 | 2024-02-15T00:08:30.031Z | INFO | SledAgent (ServiceManager): Successfully looked up all expected switch zone underlay addresses
addrs = {Switch0: fd00:1122:3344:101::2}
file = sled-agent/src/bootstrap/early_networking.rs:188
|
| 1585 | 2024-02-15T00:08:30.100Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte5", _ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 210, 225]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte5" } }
|
| 1586 | 2024-02-15T00:08:30.100Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
file = sled-agent/src/services.rs:1268
zone_type = external_dns
|
| 1587 | 2024-02-15T00:08:30.100Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:98
|
| 1588 | 2024-02-15T00:08:30.100Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:105
|
| 1589 | 2024-02-15T00:08:30.101Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:98
|
| 1590 | 2024-02-15T00:08:30.101Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:105
|
| 1591 | 2024-02-15T00:08:30.103Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.167,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:133
|
| 1592 | 2024-02-15T00:08:30.106Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1593 | 2024-02-15T00:08:30.135Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.165,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:133
|
| 1594 | 2024-02-15T00:08:30.137Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1595 | 2024-02-15T00:08:30.137Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:98
|
| 1596 | 2024-02-15T00:08:30.137Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:105
|
| 1597 | 2024-02-15T00:08:30.138Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:98
|
| 1598 | 2024-02-15T00:08:30.138Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:105
|
| 1599 | 2024-02-15T00:08:30.138Z | INFO | SledAgent (ServiceManager): no nat entry found for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:98
|
| 1600 | 2024-02-15T00:08:30.138Z | INFO | SledAgent (ServiceManager): creating nat entry for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:105
|
| 1601 | 2024-02-15T00:08:30.166Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.166,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:133
|
| 1602 | 2024-02-15T00:08:30.169Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.164,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:133
|
| 1603 | 2024-02-15T00:08:30.172Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1604 | 2024-02-15T00:08:30.172Z | INFO | SledAgent (ServiceManager): creation of nat entry successful for: V4(
Ipv4Network {
addr: 10.151.2.168,
prefix: 32,
},
)
file = clients/dpd-client/src/lib.rs:133
|
| 1605 | 2024-02-15T00:08:30.172Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1606 | 2024-02-15T00:08:30.174Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 1607 | 2024-02-15T00:08:30.175Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
file = illumos-utils/src/zone.rs:315
|
| 1608 | 2024-02-15T00:08:30.183Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
file = illumos-utils/src/zone.rs:315
|
| 1609 | 2024-02-15T00:08:30.269Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
file = illumos-utils/src/zone.rs:354
|
| 1610 | 2024-02-15T00:08:30.276Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
file = illumos-utils/src/zone.rs:354
|
| 1611 | 2024-02-15T00:08:30.281Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
file = illumos-utils/src/zone.rs:315
|
| 1612 | 2024-02-15T00:08:30.287Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
file = illumos-utils/src/zone.rs:315
|
| 1613 | 2024-02-15T00:08:30.294Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
file = illumos-utils/src/zone.rs:315
|
| 1614 | 2024-02-15T00:08:30.313Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
file = illumos-utils/src/zone.rs:354
|
| 1615 | 2024-02-15T00:08:30.321Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
file = illumos-utils/src/zone.rs:354
|
| 1616 | 2024-02-15T00:08:30.327Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
file = illumos-utils/src/zone.rs:354
|
| 1617 | 2024-02-15T00:08:33.024Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService18"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::11"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1618 | 2024-02-15T00:08:33.024Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1619 | 2024-02-15T00:08:33.070Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService22"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::f"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1620 | 2024-02-15T00:08:33.071Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1621 | 2024-02-15T00:08:33.253Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService10"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::16"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="d4c37bfe-a8de-42a7-8f5f-03d10f4b23ec"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1622 | 2024-02-15T00:08:33.253Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1623 | 2024-02-15T00:08:33.299Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService12"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::19"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="8ea64284-2ef4-4cec-aabd-94de25403e28"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1624 | 2024-02-15T00:08:33.299Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService13"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::15"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="32c8db55-cf82-4e28-b2fb-a66d32c7de65"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1625 | 2024-02-15T00:08:33.300Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService20"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::18"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="f171d330-0aca-40eb-8351-486a4f9cf306"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1626 | 2024-02-15T00:08:33.301Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService21"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::17"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="4a405f4e-6988-4f88-84f0-4359b3b74ce0"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1627 | 2024-02-15T00:08:33.301Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService23"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::14"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="d6584201-890b-4b2d-9d50-00a02ccf2ccb"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1628 | 2024-02-15T00:08:33.301Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1629 | 2024-02-15T00:08:33.312Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1630 | 2024-02-15T00:08:33.319Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService15"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::1a"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="a10818e8-180b-4e22-81ca-bb27face9611"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1631 | 2024-02-15T00:08:33.321Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService19"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::13"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="b13babdf-61fe-42c9-a2c4-58502a5da88d"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1632 | 2024-02-15T00:08:33.321Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1633 | 2024-02-15T00:08:33.328Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1634 | 2024-02-15T00:08:33.335Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1635 | 2024-02-15T00:08:33.342Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1636 | 2024-02-15T00:08:33.350Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1637 | 2024-02-15T00:08:33.438Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 32.037055ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1638 | 2024-02-15T00:08:33.449Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 58.985644ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1639 | 2024-02-15T00:08:33.485Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 148.616588ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1640 | 2024-02-15T00:08:33.544Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 123.414164ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1641 | 2024-02-15T00:08:33.565Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 65.30659ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1642 | 2024-02-15T00:08:33.608Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 66.841515ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1643 | 2024-02-15T00:08:33.623Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 58.546855ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1644 | 2024-02-15T00:08:33.638Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 33.938378ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1645 | 2024-02-15T00:08:33.654Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 27.639874ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1646 | 2024-02-15T00:08:33.674Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 27.798497ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1647 | 2024-02-15T00:08:33.726Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 134.071274ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1648 | 2024-02-15T00:08:33.764Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 212.540069ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1649 | 2024-02-15T00:08:33.789Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 65.091074ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1650 | 2024-02-15T00:08:33.809Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 75.815755ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1651 | 2024-02-15T00:08:33.841Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 158.685216ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1652 | 2024-02-15T00:08:33.875Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 102.345996ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1653 | 2024-02-15T00:08:33.904Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 60.899076ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1654 | 2024-02-15T00:08:33.943Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 291.353325ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1655 | 2024-02-15T00:08:33.979Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 70.552371ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1656 | 2024-02-15T00:08:34.119Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 462.366311ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1657 | 2024-02-15T00:08:38.472Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 120.653593ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1658 | 2024-02-15T00:08:38.614Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 105.619685ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1659 | 2024-02-15T00:08:38.634Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 71.314231ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1660 | 2024-02-15T00:08:38.650Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 324.899676ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1661 | 2024-02-15T00:08:38.670Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 117.025679ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1662 | 2024-02-15T00:08:38.683Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 140.872184ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1663 | 2024-02-15T00:08:38.703Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 225.357695ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1664 | 2024-02-15T00:08:38.716Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 224.873684ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1665 | 2024-02-15T00:08:38.746Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 144.293151ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1666 | 2024-02-15T00:08:38.754Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService17"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::10"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1667 | 2024-02-15T00:08:38.768Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 647.349496ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1668 | 2024-02-15T00:08:38.782Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 227.893795ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1669 | 2024-02-15T00:08:38.797Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 481.891782ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1670 | 2024-02-15T00:08:38.813Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 142.19313ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1671 | 2024-02-15T00:08:38.828Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 185.299599ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1672 | 2024-02-15T00:08:38.841Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 235.867784ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1673 | 2024-02-15T00:08:38.841Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1674 | 2024-02-15T00:08:38.934Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 256.724412ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1675 | 2024-02-15T00:08:38.949Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 248.783264ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1676 | 2024-02-15T00:08:38.963Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 440.85836ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1677 | 2024-02-15T00:08:38.977Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 403.873281ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1678 | 2024-02-15T00:08:38.991Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 738.51122ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1679 | 2024-02-15T00:08:39.009Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService11"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::12"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="c65199a9-aa17-4ad6-a986-8ecdb90351a7"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1680 | 2024-02-15T00:08:39.009Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1681 | 2024-02-15T00:08:39.041Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 432.415927ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1682 | 2024-02-15T00:08:39.056Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 264.695057ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1683 | 2024-02-15T00:08:39.097Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.064335836s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1684 | 2024-02-15T00:08:39.097Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1685 | 2024-02-15T00:08:39.183Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 48.812742ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1686 | 2024-02-15T00:08:39.207Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 287.084777ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1687 | 2024-02-15T00:08:39.222Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.163706339s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1688 | 2024-02-15T00:08:39.245Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 90.02551ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1689 | 2024-02-15T00:08:39.296Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 846.84572ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1690 | 2024-02-15T00:08:39.308Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 50.751462ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1691 | 2024-02-15T00:08:39.337Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 491.901484ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1692 | 2024-02-15T00:08:39.351Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 172.960439ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1693 | 2024-02-15T00:08:39.374Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 66.672119ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1694 | 2024-02-15T00:08:39.385Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 69.593638ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1695 | 2024-02-15T00:08:39.401Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 669.219575ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1696 | 2024-02-15T00:08:39.418Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.363266645s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1697 | 2024-02-15T00:08:39.437Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 755.75501ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1698 | 2024-02-15T00:08:39.454Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 172.308928ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1699 | 2024-02-15T00:08:39.468Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 69.565957ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1700 | 2024-02-15T00:08:39.490Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 825.547378ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1701 | 2024-02-15T00:08:39.509Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.162170348s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1702 | 2024-02-15T00:08:39.541Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 400.990416ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1703 | 2024-02-15T00:08:39.555Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 299.236264ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1704 | 2024-02-15T00:08:39.642Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 316.75412ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1705 | 2024-02-15T00:08:39.746Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.359952451s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1706 | 2024-02-15T00:08:39.844Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.458899489s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1707 | 2024-02-15T00:08:39.870Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 227.941601ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1708 | 2024-02-15T00:08:39.956Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 777.347576ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1709 | 2024-02-15T00:08:39.973Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 664.954791ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1710 | 2024-02-15T00:08:39.983Z | INFO | SledAgent (ServiceManager): Profile for oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService14"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::d"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/oximeter">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="id" value="a0570de6-fbf4-4073-9e84-a6e01264b6a9"/>
<propval type="astring" name="address" value="[fd00:1122:3344:101::d]:12223"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1711 | 2024-02-15T00:08:39.984Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1712 | 2024-02-15T00:08:40.030Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1713 | 2024-02-15T00:08:40.084Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.192675534s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1714 | 2024-02-15T00:08:40.111Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 978.274775ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1715 | 2024-02-15T00:08:40.156Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 716.909594ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1716 | 2024-02-15T00:08:40.175Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.06895817s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1717 | 2024-02-15T00:08:40.206Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 630.424203ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1718 | 2024-02-15T00:08:40.220Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 29.931328ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1719 | 2024-02-15T00:08:40.261Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 104.372312ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1720 | 2024-02-15T00:08:40.275Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 50.373676ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1721 | 2024-02-15T00:08:40.328Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 1.196448887s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1722 | 2024-02-15T00:08:40.337Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 99.395502ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1723 | 2024-02-15T00:08:40.377Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 242.171169ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1724 | 2024-02-15T00:08:40.398Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.047366502s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1725 | 2024-02-15T00:08:40.449Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 275.963753ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1726 | 2024-02-15T00:08:41.029Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.415182928s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1727 | 2024-02-15T00:08:41.068Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.03392668s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1728 | 2024-02-15T00:08:41.099Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 439.923649ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1729 | 2024-02-15T00:08:41.125Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 786.439681ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1730 | 2024-02-15T00:08:41.156Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 508.470683ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1731 | 2024-02-15T00:08:41.182Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.245129249s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1732 | 2024-02-15T00:08:41.224Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 585.926008ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1733 | 2024-02-15T00:08:41.241Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 543.196686ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1734 | 2024-02-15T00:08:41.261Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 465.148139ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1735 | 2024-02-15T00:08:41.276Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.08587875s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1736 | 2024-02-15T00:08:41.290Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.331737684s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1737 | 2024-02-15T00:08:41.303Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 733.419775ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1738 | 2024-02-15T00:08:41.317Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.466912347s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1739 | 2024-02-15T00:08:41.459Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 691.395736ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1740 | 2024-02-15T00:08:41.538Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 852.066953ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1741 | 2024-02-15T00:08:41.553Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 509.327485ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1742 | 2024-02-15T00:08:41.748Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.113447414s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1743 | 2024-02-15T00:08:41.764Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 483.374799ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1744 | 2024-02-15T00:08:41.801Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 1.124916293s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1745 | 2024-02-15T00:08:41.822Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 783.103504ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1746 | 2024-02-15T00:08:41.925Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.165572766s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1747 | 2024-02-15T00:08:42.050Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.21402433s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1748 | 2024-02-15T00:08:42.077Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.346707043s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1749 | 2024-02-15T00:08:42.114Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 876.022664ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1750 | 2024-02-15T00:08:42.164Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.122641688s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1751 | 2024-02-15T00:08:42.262Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.075517871s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1752 | 2024-02-15T00:08:42.376Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.404632889s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1753 | 2024-02-15T00:08:42.402Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 1.238304288s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1754 | 2024-02-15T00:08:42.439Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 968.663265ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1755 | 2024-02-15T00:08:42.459Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 836.222025ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1756 | 2024-02-15T00:08:42.619Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 661.824886ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1757 | 2024-02-15T00:08:42.634Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 804.198395ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1758 | 2024-02-15T00:08:42.798Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.24029257s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1759 | 2024-02-15T00:08:42.874Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 647.836098ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1760 | 2024-02-15T00:08:42.940Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 586.535618ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1761 | 2024-02-15T00:08:43.004Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 731.859577ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1762 | 2024-02-15T00:08:43.105Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 999.986259ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1763 | 2024-02-15T00:08:43.275Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.352549287s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1764 | 2024-02-15T00:08:43.302Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.31924644s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1765 | 2024-02-15T00:08:43.310Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 613.384286ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1766 | 2024-02-15T00:08:43.317Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.035815835s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1767 | 2024-02-15T00:08:43.346Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.403741631s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1768 | 2024-02-15T00:08:43.417Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.311777224s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1769 | 2024-02-15T00:08:43.433Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.252460213s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1770 | 2024-02-15T00:08:43.449Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.218248471s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1771 | 2024-02-15T00:08:43.531Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 516.132977ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1772 | 2024-02-15T00:08:43.539Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 1.238506056s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1773 | 2024-02-15T00:08:43.653Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 579.242393ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1774 | 2024-02-15T00:08:43.747Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.491933462s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1775 | 2024-02-15T00:08:43.792Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 966.629601ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1776 | 2024-02-15T00:08:43.936Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.15157419s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1777 | 2024-02-15T00:08:44.051Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.25321477s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1778 | 2024-02-15T00:08:44.062Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.022736949s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1779 | 2024-02-15T00:08:44.117Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.287148665s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1780 | 2024-02-15T00:08:44.245Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 910.298532ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1781 | 2024-02-15T00:08:44.362Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.497681785s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1782 | 2024-02-15T00:08:44.491Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1783 | 2024-02-15T00:08:44.496Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1784 | 2024-02-15T00:08:44.583Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1785 | 2024-02-15T00:08:44.631Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 646.765761ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1786 | 2024-02-15T00:08:44.640Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 696.711379ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1787 | 2024-02-15T00:08:44.649Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 41.674161ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1788 | 2024-02-15T00:08:44.677Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.240957212s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1789 | 2024-02-15T00:08:44.696Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 580.4933ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1790 | 2024-02-15T00:08:44.704Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 58.159465ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1791 | 2024-02-15T00:08:44.714Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 52.068417ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1792 | 2024-02-15T00:08:44.738Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 908.053983ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1793 | 2024-02-15T00:08:44.760Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 793.541867ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1794 | 2024-02-15T00:08:44.768Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 533.734155ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1795 | 2024-02-15T00:08:44.775Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 58.63086ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1796 | 2024-02-15T00:08:44.782Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 193.482351ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1797 | 2024-02-15T00:08:44.790Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 1.454076534s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1798 | 2024-02-15T00:08:44.843Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 171.777682ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1799 | 2024-02-15T00:08:44.987Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 242.723566ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1800 | 2024-02-15T00:08:45.027Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 324.467772ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1801 | 2024-02-15T00:08:45.041Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 26.168566ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1802 | 2024-02-15T00:08:45.076Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 129.066634ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1803 | 2024-02-15T00:08:45.093Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 884.460449ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1804 | 2024-02-15T00:08:45.101Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.011257001s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1805 | 2024-02-15T00:08:45.165Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 944.733709ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1806 | 2024-02-15T00:08:45.213Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 164.179575ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1807 | 2024-02-15T00:08:45.238Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 438.404344ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1808 | 2024-02-15T00:08:45.249Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.093118s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1809 | 2024-02-15T00:08:45.285Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.320219305s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1810 | 2024-02-15T00:08:45.294Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.128620754s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1811 | 2024-02-15T00:08:45.311Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 512.584886ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1812 | 2024-02-15T00:08:45.319Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 750.407367ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1813 | 2024-02-15T00:08:45.345Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.456865159s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1814 | 2024-02-15T00:08:45.359Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 708.064627ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1815 | 2024-02-15T00:08:45.385Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 403.2817ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1816 | 2024-02-15T00:08:45.412Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 637.318844ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1817 | 2024-02-15T00:08:45.564Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 664.972731ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1818 | 2024-02-15T00:08:45.656Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.232217871s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1819 | 2024-02-15T00:08:45.686Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.17096783s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1820 | 2024-02-15T00:08:45.798Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 506.39045ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1821 | 2024-02-15T00:08:45.833Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.345311176s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1822 | 2024-02-15T00:08:45.869Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 930.578863ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1823 | 2024-02-15T00:08:45.926Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 912.657253ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1824 | 2024-02-15T00:08:45.985Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.119081535s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1825 | 2024-02-15T00:08:46.060Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.167183493s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1826 | 2024-02-15T00:08:46.077Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.196655613s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1827 | 2024-02-15T00:08:46.086Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 975.091503ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1828 | 2024-02-15T00:08:46.118Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 978.946251ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1829 | 2024-02-15T00:08:46.126Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.266943796s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1830 | 2024-02-15T00:08:46.240Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 871.707861ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1831 | 2024-02-15T00:08:46.256Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 926.087236ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1832 | 2024-02-15T00:08:46.315Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 976.919801ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1833 | 2024-02-15T00:08:46.352Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.220380428s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1834 | 2024-02-15T00:08:46.432Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 727.797405ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1835 | 2024-02-15T00:08:46.615Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 862.26684ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1836 | 2024-02-15T00:08:46.809Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 525.176712ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1837 | 2024-02-15T00:08:46.818Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 657.842738ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1838 | 2024-02-15T00:08:46.848Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 554.903331ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1839 | 2024-02-15T00:08:46.866Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.263336291s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1840 | 2024-02-15T00:08:46.896Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 514.796858ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1841 | 2024-02-15T00:08:47.070Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.125647407s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1842 | 2024-02-15T00:08:47.118Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 510.561069ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1843 | 2024-02-15T00:08:47.127Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 711.359446ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1844 | 2024-02-15T00:08:47.135Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.456669826s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1845 | 2024-02-15T00:08:47.168Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.330662075s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1846 | 2024-02-15T00:08:47.187Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.407771908s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1847 | 2024-02-15T00:08:47.195Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 1.413720748s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1848 | 2024-02-15T00:08:47.236Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 555.264792ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1849 | 2024-02-15T00:08:47.283Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 878.347316ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1850 | 2024-02-15T00:08:47.302Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.28751179s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1851 | 2024-02-15T00:08:47.344Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 557.647324ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1852 | 2024-02-15T00:08:47.401Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 762.064153ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1853 | 2024-02-15T00:08:47.412Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 664.365502ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1854 | 2024-02-15T00:08:47.420Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.174762579s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1855 | 2024-02-15T00:08:47.485Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.325515441s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1856 | 2024-02-15T00:08:47.492Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 550.818773ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1857 | 2024-02-15T00:08:47.581Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 825.755843ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1858 | 2024-02-15T00:08:47.637Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 629.415335ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1859 | 2024-02-15T00:08:47.802Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 554.4028ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1860 | 2024-02-15T00:08:47.847Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.421420534s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1861 | 2024-02-15T00:08:47.911Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 611.313776ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1862 | 2024-02-15T00:08:48.053Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.406251761s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1863 | 2024-02-15T00:08:48.085Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 659.19332ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1864 | 2024-02-15T00:08:48.139Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.218509117s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1865 | 2024-02-15T00:08:48.170Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.195632583s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1866 | 2024-02-15T00:08:48.178Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 686.590018ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1867 | 2024-02-15T00:08:48.205Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 952.516529ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1868 | 2024-02-15T00:08:48.276Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 1.415529554s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1869 | 2024-02-15T00:08:48.365Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 690.607339ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1870 | 2024-02-15T00:08:48.416Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 880.019434ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1871 | 2024-02-15T00:08:48.508Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.171104522s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1872 | 2024-02-15T00:08:48.531Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 912.6091ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1873 | 2024-02-15T00:08:48.599Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.271238991s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1874 | 2024-02-15T00:08:48.607Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.357407385s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1875 | 2024-02-15T00:08:48.614Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.103327978s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1876 | 2024-02-15T00:08:48.622Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 664.970508ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1877 | 2024-02-15T00:08:48.631Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 1.119827941s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1878 | 2024-02-15T00:08:48.755Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 960.420487ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1879 | 2024-02-15T00:08:48.820Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 551.124167ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1880 | 2024-02-15T00:08:48.875Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 796.927866ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1881 | 2024-02-15T00:08:49.067Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.073129276s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1882 | 2024-02-15T00:08:49.167Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 505.205407ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1883 | 2024-02-15T00:08:49.278Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 637.048388ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1884 | 2024-02-15T00:08:49.297Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.18983857s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1885 | 2024-02-15T00:08:49.304Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 549.395458ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1886 | 2024-02-15T00:08:49.366Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.283506603s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1887 | 2024-02-15T00:08:49.375Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.282031626s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1888 | 2024-02-15T00:08:49.382Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 971.925544ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1889 | 2024-02-15T00:08:49.454Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 883.116128ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1890 | 2024-02-15T00:08:49.480Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.462252034s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1891 | 2024-02-15T00:08:49.698Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.211835745s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1892 | 2024-02-15T00:08:49.714Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.067527708s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1893 | 2024-02-15T00:08:49.725Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.129035726s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1894 | 2024-02-15T00:08:49.735Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 808.885606ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1895 | 2024-02-15T00:08:49.746Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.470956289s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1896 | 2024-02-15T00:08:49.756Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.117961295s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1897 | 2024-02-15T00:08:49.766Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 825.977266ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1898 | 2024-02-15T00:08:49.864Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 712.137249ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1899 | 2024-02-15T00:08:49.880Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 689.572808ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1900 | 2024-02-15T00:08:49.924Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.096961989s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1901 | 2024-02-15T00:08:49.975Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.138029526s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1902 | 2024-02-15T00:08:50.150Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 809.008992ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1903 | 2024-02-15T00:08:50.346Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.313491542s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1904 | 2024-02-15T00:08:50.365Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.136788863s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1905 | 2024-02-15T00:08:50.496Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.086337037s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1906 | 2024-02-15T00:08:50.553Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 660.208732ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1907 | 2024-02-15T00:08:50.578Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 849.484057ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1908 | 2024-02-15T00:08:50.586Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.398031463s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1909 | 2024-02-15T00:08:50.602Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 915.07971ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1910 | 2024-02-15T00:08:50.659Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.050972694s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1911 | 2024-02-15T00:08:50.667Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.1774151s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1912 | 2024-02-15T00:08:50.790Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 842.456863ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1913 | 2024-02-15T00:08:50.867Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.493312684s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1914 | 2024-02-15T00:08:50.888Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 812.288354ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1915 | 2024-02-15T00:08:50.921Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 540.079037ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1916 | 2024-02-15T00:08:50.954Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 788.964485ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1917 | 2024-02-15T00:08:50.972Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 805.921296ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1918 | 2024-02-15T00:08:51.035Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 845.893534ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1919 | 2024-02-15T00:08:51.123Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.462029665s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1920 | 2024-02-15T00:08:51.221Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 1.419708587s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1921 | 2024-02-15T00:08:51.229Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.187179437s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1922 | 2024-02-15T00:08:51.259Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:59693
|
| 1923 | 2024-02-15T00:08:51.327Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 67677
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:59693
req_id = ca81bf78-fd66-4ce7-b916-9235890f01cd
response_code = 200
uri = /baseboard
|
| 1924 | 2024-02-15T00:08:51.439Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 560.382017ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1925 | 2024-02-15T00:08:51.471Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 864.923512ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1926 | 2024-02-15T00:08:51.511Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.236499634s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1927 | 2024-02-15T00:08:51.525Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 532.650073ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1928 | 2024-02-15T00:08:51.592Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.393607539s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1929 | 2024-02-15T00:08:51.642Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.237322261s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1930 | 2024-02-15T00:08:51.668Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 762.324382ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1931 | 2024-02-15T00:08:51.711Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 680.292285ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1932 | 2024-02-15T00:08:51.711Z | INFO | SledAgent (ServiceManager): Profile for oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/zone-network-setup">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService16"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="static_addr" value="fd00:1122:3344:101::e"/>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
<service version="1" type="service" name="network/ssh">
<instance enabled="false" name="default">
</instance>
</service>
<service version="1" type="service" name="oxide/clickhouse">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::e"/>
<propval type="astring" name="listen_port" value="8123"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
<service version="1" type="service" name="network/dns/install">
<property_group type="application" name="install_props">
<property type="net_address" name="nameserver">
<net_address_list>
<value_node value="fd00:1122:3344:3::1" />
<value_node value="fd00:1122:3344:2::1" />
<value_node value="fd00:1122:3344:1::1" />
</net_address_list>
</property>
</property_group>
<instance enabled="true" name="default">
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
|
| 1933 | 2024-02-15T00:08:51.720Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.28406776s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1934 | 2024-02-15T00:08:51.720Z | INFO | SledAgent (ServiceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 1935 | 2024-02-15T00:08:51.754Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.009690136s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1936 | 2024-02-15T00:08:51.789Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 639.55965ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1937 | 2024-02-15T00:08:51.855Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.341612584s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1938 | 2024-02-15T00:08:51.892Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 577.94168ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1939 | 2024-02-15T00:08:51.903Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 42.802677ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 1940 | 2024-02-15T00:08:51.954Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 129.12156ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 1941 | 2024-02-15T00:08:51.993Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 971.549856ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1942 | 2024-02-15T00:08:52.007Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.351075044s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1943 | 2024-02-15T00:08:52.070Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 671.326307ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1944 | 2024-02-15T00:08:52.105Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 234.861531ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 1945 | 2024-02-15T00:08:52.347Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 703.21809ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1946 | 2024-02-15T00:08:52.356Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 468.319824ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 1947 | 2024-02-15T00:08:52.370Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.344721238s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1948 | 2024-02-15T00:08:52.400Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.262406989s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1949 | 2024-02-15T00:08:52.426Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 721.734838ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1950 | 2024-02-15T00:08:52.438Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.043459537s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1951 | 2024-02-15T00:08:52.446Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 846.175862ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1952 | 2024-02-15T00:08:52.479Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.377701119s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1953 | 2024-02-15T00:08:52.595Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 712.007193ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1954 | 2024-02-15T00:08:52.650Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 680.257857ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1955 | 2024-02-15T00:08:52.751Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 940.77057ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1956 | 2024-02-15T00:08:52.759Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.27168745s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1957 | 2024-02-15T00:08:52.773Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.258844318s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1958 | 2024-02-15T00:08:52.835Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 533.223737ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 1959 | 2024-02-15T00:08:52.890Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.401508479s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1960 | 2024-02-15T00:08:52.974Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.220924152s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1961 | 2024-02-15T00:08:52.995Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 926.094085ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1962 | 2024-02-15T00:08:53.015Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.13912863s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1963 | 2024-02-15T00:08:53.059Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 638.060724ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1964 | 2024-02-15T00:08:53.157Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.124572671s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1965 | 2024-02-15T00:08:53.206Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 731.69469ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1966 | 2024-02-15T00:08:53.303Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 815.453491ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1967 | 2024-02-15T00:08:53.317Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 926.171457ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1968 | 2024-02-15T00:08:53.340Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 618.13214ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1969 | 2024-02-15T00:08:53.368Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.061827103s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1970 | 2024-02-15T00:08:53.377Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 1.127909364s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 1971 | 2024-02-15T00:08:53.491Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.066601201s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1972 | 2024-02-15T00:08:53.673Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.279491731s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1973 | 2024-02-15T00:08:53.701Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 919.616237ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1974 | 2024-02-15T00:08:53.708Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.427321661s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 1975 | 2024-02-15T00:08:53.722Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 697.814523ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1976 | 2024-02-15T00:08:53.867Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 827.460254ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1977 | 2024-02-15T00:08:53.932Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 546.192712ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1978 | 2024-02-15T00:08:53.946Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 579.332961ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1979 | 2024-02-15T00:08:53.967Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 824.909457ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1980 | 2024-02-15T00:08:54.041Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.067066881s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 1981 | 2024-02-15T00:08:54.050Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 549.23633ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1982 | 2024-02-15T00:08:54.129Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.025076491s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 1983 | 2024-02-15T00:08:54.163Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 936.795058ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 1984 | 2024-02-15T00:08:54.203Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.093579075s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 1985 | 2024-02-15T00:08:54.251Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 841.599867ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 1986 | 2024-02-15T00:08:54.292Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 772.527694ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 1987 | 2024-02-15T00:08:54.301Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.014111458s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 1988 | 2024-02-15T00:08:54.430Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.091046816s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 1989 | 2024-02-15T00:08:54.439Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.463630612s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 1990 | 2024-02-15T00:08:54.490Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.335429127s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 1991 | 2024-02-15T00:08:54.514Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 1.439118529s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 1992 | 2024-02-15T00:08:54.536Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.240379484s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 1993 | 2024-02-15T00:08:54.567Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 835.558276ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 1994 | 2024-02-15T00:08:54.609Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 618.113081ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 1995 | 2024-02-15T00:08:54.630Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 770.723776ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 1996 | 2024-02-15T00:08:54.703Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 593.218857ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 1997 | 2024-02-15T00:08:54.801Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 1.068891066s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 1998 | 2024-02-15T00:08:54.963Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 608.66535ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 1999 | 2024-02-15T00:08:55.075Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 768.98734ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2000 | 2024-02-15T00:08:55.103Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.178317233s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2001 | 2024-02-15T00:08:55.112Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 792.105558ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2002 | 2024-02-15T00:08:55.121Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.402665193s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 2003 | 2024-02-15T00:08:55.145Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 946.982878ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 2004 | 2024-02-15T00:08:55.163Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.264092035s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 2005 | 2024-02-15T00:08:55.237Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.428385712s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 2006 | 2024-02-15T00:08:55.306Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.258354848s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2007 | 2024-02-15T00:08:55.315Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 545.237952ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2008 | 2024-02-15T00:08:55.325Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.339844863s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 2009 | 2024-02-15T00:08:55.410Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 924.365852ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2010 | 2024-02-15T00:08:55.417Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.04152536s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 2011 | 2024-02-15T00:08:55.531Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 810.018964ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2012 | 2024-02-15T00:08:55.581Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.343156255s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2013 | 2024-02-15T00:08:55.787Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.145388908s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2014 | 2024-02-15T00:08:55.835Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 725.946888ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 2015 | 2024-02-15T00:08:55.852Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.05565334s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2016 | 2024-02-15T00:08:55.869Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 631.030074ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2017 | 2024-02-15T00:08:55.878Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 1.485622019s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 2018 | 2024-02-15T00:08:55.910Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.444486542s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2019 | 2024-02-15T00:08:55.917Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.478924782s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2020 | 2024-02-15T00:08:55.962Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 1.170973797s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 2021 | 2024-02-15T00:08:56.104Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.469003904s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 2022 | 2024-02-15T00:08:56.292Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 819.385509ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2023 | 2024-02-15T00:08:56.343Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 688.439219ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2024 | 2024-02-15T00:08:56.350Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.181737716s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2025 | 2024-02-15T00:08:56.437Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.248426705s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 2026 | 2024-02-15T00:08:56.467Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.132354256s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 2027 | 2024-02-15T00:08:56.509Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.067736044s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2028 | 2024-02-15T00:08:56.533Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.069769471s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 2029 | 2024-02-15T00:08:56.570Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.043468524s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 2030 | 2024-02-15T00:08:56.579Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.445583964s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2031 | 2024-02-15T00:08:56.677Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 764.956216ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 2032 | 2024-02-15T00:08:56.686Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.400482338s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 2033 | 2024-02-15T00:08:56.920Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 915.696845ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2034 | 2024-02-15T00:08:56.935Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.347402361s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2035 | 2024-02-15T00:08:56.945Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 936.418016ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2036 | 2024-02-15T00:08:57.044Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 1.314007549s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2037 | 2024-02-15T00:08:57.123Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 675.722545ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2038 | 2024-02-15T00:08:57.144Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 1.168711966s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 2039 | 2024-02-15T00:08:57.366Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.185685346s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2040 | 2024-02-15T00:08:57.375Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 1.463762458s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 2041 | 2024-02-15T00:08:57.406Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.132983173s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2042 | 2024-02-15T00:08:57.451Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.128273508s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 2043 | 2024-02-15T00:08:57.543Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 722.110367ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2044 | 2024-02-15T00:08:57.584Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 503.325944ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 2045 | 2024-02-15T00:08:57.594Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 614.53544ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2046 | 2024-02-15T00:08:57.609Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.062237672s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 2047 | 2024-02-15T00:08:57.617Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 937.807802ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 2048 | 2024-02-15T00:08:57.628Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.151744899s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 2049 | 2024-02-15T00:08:57.698Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 941.741419ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 2050 | 2024-02-15T00:08:57.813Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.016525269s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2051 | 2024-02-15T00:08:57.848Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 530.121329ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2052 | 2024-02-15T00:08:57.893Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.363546025s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2053 | 2024-02-15T00:08:58.037Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 595.787671ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2054 | 2024-02-15T00:08:58.099Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.000877506s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 2055 | 2024-02-15T00:08:58.109Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.393279558s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 2056 | 2024-02-15T00:08:58.219Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 861.429873ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2057 | 2024-02-15T00:08:58.276Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.158404023s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2058 | 2024-02-15T00:08:58.292Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 894.149996ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2059 | 2024-02-15T00:08:58.320Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 611.887835ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 2060 | 2024-02-15T00:08:58.366Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 735.753914ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2061 | 2024-02-15T00:08:58.385Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 502.582084ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2062 | 2024-02-15T00:08:58.550Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.351567799s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2063 | 2024-02-15T00:08:58.563Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.371609268s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2064 | 2024-02-15T00:08:58.570Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.416807131s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 2065 | 2024-02-15T00:08:58.588Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.340721633s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 2066 | 2024-02-15T00:08:58.645Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 832.876155ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2067 | 2024-02-15T00:08:58.655Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 800.864172ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 2068 | 2024-02-15T00:08:58.682Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 597.106997ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 2069 | 2024-02-15T00:08:58.791Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 971.399702ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 2070 | 2024-02-15T00:08:58.839Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 502.842422ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2071 | 2024-02-15T00:08:58.849Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 1.14468202s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 2072 | 2024-02-15T00:08:58.898Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.399253314s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2073 | 2024-02-15T00:08:58.942Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 1.298255641s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 2074 | 2024-02-15T00:08:59.090Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.129591932s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2075 | 2024-02-15T00:08:59.108Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.256116721s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 2076 | 2024-02-15T00:08:59.116Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 851.413204ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2077 | 2024-02-15T00:08:59.197Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 516.229556ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2078 | 2024-02-15T00:08:59.267Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 945.740837ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2079 | 2024-02-15T00:08:59.288Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.48715812s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 2080 | 2024-02-15T00:08:59.352Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 954.006743ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2081 | 2024-02-15T00:08:59.447Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.063419561s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2082 | 2024-02-15T00:08:59.471Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.060437171s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 2083 | 2024-02-15T00:08:59.491Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 553.650676ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2084 | 2024-02-15T00:08:59.514Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.448691114s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 2085 | 2024-02-15T00:08:59.724Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 512.210787ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2086 | 2024-02-15T00:08:59.773Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 741.469426ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 2087 | 2024-02-15T00:08:59.911Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 632.484571ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2088 | 2024-02-15T00:08:59.940Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 973.86899ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 2089 | 2024-02-15T00:08:59.949Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.367301682s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2090 | 2024-02-15T00:08:59.978Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 850.737186ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2091 | 2024-02-15T00:08:59.997Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.056586858s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 2092 | 2024-02-15T00:09:00.005Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 836.239558ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 2093 | 2024-02-15T00:09:00.057Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.248055596s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2094 | 2024-02-15T00:09:00.225Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.085733361s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2095 | 2024-02-15T00:09:00.234Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.422388611s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2096 | 2024-02-15T00:09:00.247Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.068851741s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2097 | 2024-02-15T00:09:00.256Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 819.509842ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 2098 | 2024-02-15T00:09:00.308Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.379501266s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2099 | 2024-02-15T00:09:00.320Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.356517467s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2100 | 2024-02-15T00:09:00.379Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.161211546s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 2101 | 2024-02-15T00:09:00.523Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 616.295824ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2102 | 2024-02-15T00:09:00.535Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.45473491s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 2103 | 2024-02-15T00:09:00.546Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 1.419009732s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 2104 | 2024-02-15T00:09:00.556Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.406072261s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2105 | 2024-02-15T00:09:00.792Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.476194001s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 2106 | 2024-02-15T00:09:00.843Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 1.307183543s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2107 | 2024-02-15T00:09:00.856Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 942.709477ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 2108 | 2024-02-15T00:09:00.934Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 936.924028ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 2109 | 2024-02-15T00:09:00.982Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 575.619461ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 2110 | 2024-02-15T00:09:01.071Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.289598451s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 2111 | 2024-02-15T00:09:01.094Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 1.322302485s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 2112 | 2024-02-15T00:09:01.164Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 643.121869ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2113 | 2024-02-15T00:09:01.325Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 658.044454ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2114 | 2024-02-15T00:09:01.342Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.152705886s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2115 | 2024-02-15T00:09:01.364Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.271520549s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2116 | 2024-02-15T00:09:01.381Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 540.886917ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2117 | 2024-02-15T00:09:01.580Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.226755764s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 2118 | 2024-02-15T00:09:01.600Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.284423305s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 2119 | 2024-02-15T00:09:01.679Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 802.136681ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2120 | 2024-02-15T00:09:01.704Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 527.144345ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2121 | 2024-02-15T00:09:01.729Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.033165778s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2122 | 2024-02-15T00:09:01.821Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 1.114733098s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 2123 | 2024-02-15T00:09:01.840Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.24833896s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2124 | 2024-02-15T00:09:01.888Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.157863032s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 2125 | 2024-02-15T00:09:01.940Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.478129387s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2126 | 2024-02-15T00:09:01.985Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.345218971s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2127 | 2024-02-15T00:09:02.007Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 975.813252ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 2128 | 2024-02-15T00:09:02.026Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.495521379s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2129 | 2024-02-15T00:09:02.049Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 873.988529ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 2130 | 2024-02-15T00:09:02.168Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 889.743322ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2131 | 2024-02-15T00:09:02.249Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 870.486579ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2132 | 2024-02-15T00:09:02.286Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 580.441928ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 2133 | 2024-02-15T00:09:02.375Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.490371861s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 2134 | 2024-02-15T00:09:02.432Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 1.16333641s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 2135 | 2024-02-15T00:09:02.494Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 848.137997ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2136 | 2024-02-15T00:09:02.507Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 866.673856ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2137 | 2024-02-15T00:09:02.652Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 1.323491373s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2138 | 2024-02-15T00:09:02.775Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.210597425s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2139 | 2024-02-15T00:09:02.820Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 1.145185094s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 2140 | 2024-02-15T00:09:02.893Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.4166746s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 2141 | 2024-02-15T00:09:02.904Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.189810403s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 2142 | 2024-02-15T00:09:02.933Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 819.563849ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 2143 | 2024-02-15T00:09:02.945Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 636.3715ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 2144 | 2024-02-15T00:09:02.993Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 824.014571ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 2145 | 2024-02-15T00:09:03.058Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 700.650828ms
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 2146 | 2024-02-15T00:09:03.069Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 1.131178698s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2147 | 2024-02-15T00:09:03.100Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 1.49948274s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2148 | 2024-02-15T00:09:03.129Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 542.816453ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2149 | 2024-02-15T00:09:03.343Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3") failed: Property not found. retry in 1.329223086s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2150 | 2024-02-15T00:09:03.354Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 538.253504ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2151 | 2024-02-15T00:09:03.386Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 639.604301ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2152 | 2024-02-15T00:09:03.430Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 613.93729ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2153 | 2024-02-15T00:09:03.534Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 584.870702ms
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2154 | 2024-02-15T00:09:03.594Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21") failed: Property not found. retry in 1.331149377s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 2155 | 2024-02-15T00:09:03.606Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 1.076352977s
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 2156 | 2024-02-15T00:09:03.684Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97") failed: Property not found. retry in 1.047205832s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2157 | 2024-02-15T00:09:03.764Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21") failed: Property not found. retry in 1.418893603s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 2158 | 2024-02-15T00:09:03.774Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9") failed: Property not found. retry in 1.133398827s
file = illumos-utils/src/svc.rs:36
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 2159 | 2024-02-15T00:09:03.828Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566") failed: Property not found. retry in 903.20267ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 2160 | 2024-02-15T00:09:03.878Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa") failed: Property not found. retry in 1.411042535s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 2161 | 2024-02-15T00:09:03.904Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c") failed: Property not found. retry in 1.403151427s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2162 | 2024-02-15T00:09:03.977Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329") failed: Property not found. retry in 880.907063ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 2163 | 2024-02-15T00:09:03.986Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5") failed: Property not found. retry in 859.530033ms
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2164 | 2024-02-15T00:09:04.007Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e") failed: Property not found. retry in 1.279111742s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2165 | 2024-02-15T00:09:04.038Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c") failed: Property not found. retry in 1.187765128s
file = illumos-utils/src/svc.rs:36
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2166 | 2024-02-15T00:09:04.060Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab") failed: Property not found. retry in 1.344120743s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2167 | 2024-02-15T00:09:04.105Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d") failed: Property not found. retry in 1.417818539s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 2168 | 2024-02-15T00:09:04.129Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69") failed: Property not found. retry in 1.435870404s
file = illumos-utils/src/svc.rs:36
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2169 | 2024-02-15T00:09:04.211Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7") failed: Property not found. retry in 745.730885ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2170 | 2024-02-15T00:09:04.321Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35") failed: Property not found. retry in 1.287240767s
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 2171 | 2024-02-15T00:09:04.611Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4") failed: Property not found. retry in 505.122831ms
file = illumos-utils/src/svc.rs:36
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2172 | 2024-02-15T00:09:04.697Z | WARN | SledAgent (ServiceManager): wait for service Some("oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c") failed: Property not found. retry in 744.156685ms
file = illumos-utils/src/svc.rs:36
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 2173 | 2024-02-15T00:09:04.727Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::c exists
file = sled-agent/src/services.rs:1900
|
| 2174 | 2024-02-15T00:09:04.728Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::c, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2175 | 2024-02-15T00:09:05.301Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::c exists - OK
file = sled-agent/src/services.rs:1908
|
| 2176 | 2024-02-15T00:09:05.301Z | INFO | SledAgent (ServiceManager): Zone using sled underlay as gateway
file = sled-agent/src/services.rs:1927
|
| 2177 | 2024-02-15T00:09:09.017Z | INFO | SledAgent (ServiceManager): Setting up Nexus service
file = sled-agent/src/services.rs:1971
|
| 2178 | 2024-02-15T00:09:09.017Z | INFO | SledAgent (ServiceManager): Ensuring address for OPTE port
file = illumos-utils/src/running_zone.rs:653
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2179 | 2024-02-15T00:09:14.929Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::b exists
file = sled-agent/src/services.rs:1900
|
| 2180 | 2024-02-15T00:09:14.929Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::b, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2181 | 2024-02-15T00:09:15.482Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::b exists - OK
file = sled-agent/src/services.rs:1908
|
| 2182 | 2024-02-15T00:09:15.482Z | INFO | SledAgent (ServiceManager): Zone using sled underlay as gateway
file = sled-agent/src/services.rs:1927
|
| 2183 | 2024-02-15T00:09:15.699Z | INFO | SledAgent (ServiceManager): Setting up Nexus service
file = sled-agent/src/services.rs:1971
|
| 2184 | 2024-02-15T00:09:15.699Z | INFO | SledAgent (ServiceManager): Ensuring address for OPTE port
file = illumos-utils/src/running_zone.rs:653
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2185 | 2024-02-15T00:09:21.364Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:51490
|
| 2186 | 2024-02-15T00:09:21.364Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 80
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:51490
req_id = 9de83871-36f1-4c83-8ce4-96b487c77122
response_code = 200
uri = /baseboard
|
| 2187 | 2024-02-15T00:09:22.066Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::9 exists
file = sled-agent/src/services.rs:1900
|
| 2188 | 2024-02-15T00:09:22.066Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::9, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2189 | 2024-02-15T00:09:22.622Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::9 exists - OK
file = sled-agent/src/services.rs:1908
|
| 2190 | 2024-02-15T00:09:22.622Z | INFO | SledAgent (ServiceManager): Zone using sled underlay as gateway
file = sled-agent/src/services.rs:1927
|
| 2191 | 2024-02-15T00:09:22.833Z | INFO | SledAgent (ServiceManager): Setting up external-dns service
file = sled-agent/src/services.rs:2084
|
| 2192 | 2024-02-15T00:09:22.833Z | INFO | SledAgent (ServiceManager): Ensuring address for OPTE port
file = illumos-utils/src/running_zone.rs:653
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2193 | 2024-02-15T00:09:28.365Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::a exists
file = sled-agent/src/services.rs:1900
|
| 2194 | 2024-02-15T00:09:28.365Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::a, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2195 | 2024-02-15T00:09:28.921Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 243.107168304s
|
| 2196 | 2024-02-15T00:09:28.923Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 243.110558332s
|
| 2197 | 2024-02-15T00:09:28.923Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 243.11048233s
|
| 2198 | 2024-02-15T00:09:28.924Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::a exists - OK
file = sled-agent/src/services.rs:1908
|
| 2199 | 2024-02-15T00:09:28.924Z | INFO | SledAgent (ServiceManager): Zone using sled underlay as gateway
file = sled-agent/src/services.rs:1927
|
| 2200 | 2024-02-15T00:09:29.010Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 243.198925594s
|
| 2201 | 2024-02-15T00:09:29.161Z | INFO | SledAgent (ServiceManager): Setting up Nexus service
file = sled-agent/src/services.rs:1971
|
| 2202 | 2024-02-15T00:09:29.161Z | INFO | SledAgent (ServiceManager): Ensuring address for OPTE port
file = illumos-utils/src/running_zone.rs:653
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2203 | 2024-02-15T00:09:29.740Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 243.928712323s
|
| 2204 | 2024-02-15T00:09:31.518Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 245.704005845s
|
| 2205 | 2024-02-15T00:09:31.590Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/sled_agent.rs:741
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
total duration = 235.468961749s
|
| 2206 | 2024-02-15T00:09:31.591Z | WARN | SledAgent: failed to register as a metric producer with Nexus
error = "Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/sled_agent.rs:1199
retry_after = 22.128172115s
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2207 | 2024-02-15T00:09:31.591Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 245.777474069s
|
| 2208 | 2024-02-15T00:09:31.592Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 245.777723245s
|
| 2209 | 2024-02-15T00:09:31.592Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 245.779839987s
|
| 2210 | 2024-02-15T00:09:31.592Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 245.780618673s
|
| 2211 | 2024-02-15T00:09:31.592Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 245.781153303s
|
| 2212 | 2024-02-15T00:09:31.592Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 245.778965229s
|
| 2213 | 2024-02-15T00:09:31.600Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 245.788596312s
|
| 2214 | 2024-02-15T00:09:31.601Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 245.789557571s
|
| 2215 | 2024-02-15T00:09:31.601Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 245.78902958s
|
| 2216 | 2024-02-15T00:09:31.601Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 245.788486839s
|
| 2217 | 2024-02-15T00:09:31.601Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 245.787821156s
|
| 2218 | 2024-02-15T00:09:31.601Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 245.788456579s
|
| 2219 | 2024-02-15T00:09:31.601Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 245.79104496s
|
| 2220 | 2024-02-15T00:09:31.602Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 245.788125652s
|
| 2221 | 2024-02-15T00:09:32.341Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 246.529746217s
|
| 2222 | 2024-02-15T00:09:32.471Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 246.659217608s
|
| 2223 | 2024-02-15T00:09:32.843Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 247.031357377s
|
| 2224 | 2024-02-15T00:09:33.150Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 247.338089912s
|
| 2225 | 2024-02-15T00:09:33.151Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 247.33848847s
|
| 2226 | 2024-02-15T00:09:33.151Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 247.336826988s
|
| 2227 | 2024-02-15T00:09:33.485Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2228 | 2024-02-15T00:09:33.488Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/sled_agent.rs:741
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
total duration = 237.366267673s
|
| 2229 | 2024-02-15T00:09:34.652Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 248.840044125s
|
| 2230 | 2024-02-15T00:09:34.716Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 248.902364957s
|
| 2231 | 2024-02-15T00:09:34.956Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 249.143398493s
|
| 2232 | 2024-02-15T00:09:35.281Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 249.468998364s
|
| 2233 | 2024-02-15T00:09:35.647Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2234 | 2024-02-15T00:09:35.655Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/sled_agent.rs:741
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
total duration = 239.533315956s
|
| 2235 | 2024-02-15T00:09:35.673Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::8 exists
file = sled-agent/src/services.rs:1900
|
| 2236 | 2024-02-15T00:09:35.673Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::8, prefix: 64 }))
file = illumos-utils/src/running_zone.rs:609
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2237 | 2024-02-15T00:09:36.220Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 250.409469523s
|
| 2238 | 2024-02-15T00:09:36.228Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::8 exists - OK
file = sled-agent/src/services.rs:1908
|
| 2239 | 2024-02-15T00:09:36.228Z | INFO | SledAgent (ServiceManager): Zone using sled underlay as gateway
file = sled-agent/src/services.rs:1927
|
| 2240 | 2024-02-15T00:09:36.425Z | INFO | SledAgent (ServiceManager): Setting up external-dns service
file = sled-agent/src/services.rs:2084
|
| 2241 | 2024-02-15T00:09:36.425Z | INFO | SledAgent (ServiceManager): Ensuring address for OPTE port
file = illumos-utils/src/running_zone.rs:653
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2242 | 2024-02-15T00:09:36.450Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 250.638649303s
|
| 2243 | 2024-02-15T00:09:36.561Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 250.749702946s
|
| 2244 | 2024-02-15T00:09:36.685Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 250.871694179s
|
| 2245 | 2024-02-15T00:09:36.847Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 251.034567965s
|
| 2246 | 2024-02-15T00:09:36.874Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 251.063794777s
|
| 2247 | 2024-02-15T00:09:36.942Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 251.12960956s
|
| 2248 | 2024-02-15T00:09:37.076Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 251.263368836s
|
| 2249 | 2024-02-15T00:09:37.372Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 251.560641193s
|
| 2250 | 2024-02-15T00:09:37.458Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 251.646448774s
|
| 2251 | 2024-02-15T00:09:37.573Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 251.760878795s
|
| 2252 | 2024-02-15T00:09:37.645Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 251.83235657s
|
| 2253 | 2024-02-15T00:09:37.938Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 252.124741669s
|
| 2254 | 2024-02-15T00:09:37.957Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 252.142921802s
|
| 2255 | 2024-02-15T00:09:38.226Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 252.413957595s
|
| 2256 | 2024-02-15T00:09:38.486Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 252.671902027s
|
| 2257 | 2024-02-15T00:09:38.523Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 252.709880244s
|
| 2258 | 2024-02-15T00:09:38.606Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2259 | 2024-02-15T00:09:38.608Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/sled_agent.rs:741
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
total duration = 242.486902318s
|
| 2260 | 2024-02-15T00:09:38.721Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 252.909488274s
|
| 2261 | 2024-02-15T00:09:38.886Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 253.072084435s
|
| 2262 | 2024-02-15T00:09:38.972Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 253.16058351s
|
| 2263 | 2024-02-15T00:09:39.157Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 253.345772492s
|
| 2264 | 2024-02-15T00:09:39.906Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 254.093984668s
|
| 2265 | 2024-02-15T00:09:40.034Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 254.221804846s
|
| 2266 | 2024-02-15T00:09:41.087Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 255.27437582s
|
| 2267 | 2024-02-15T00:09:41.141Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 255.329633642s
|
| 2268 | 2024-02-15T00:09:41.223Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 255.410964333s
|
| 2269 | 2024-02-15T00:09:41.266Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 255.455153374s
|
| 2270 | 2024-02-15T00:09:41.356Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 255.543516436s
|
| 2271 | 2024-02-15T00:09:41.605Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_nexus_052b5be3-98fd-4823-9614-96d5055e333c
|
| 2272 | 2024-02-15T00:09:41.605Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_c7373a32-5911-4b61-b967-b9351fe9441e
|
| 2273 | 2024-02-15T00:09:41.605Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_c22ce43b-b4c5-453b-9120-505102876329
|
| 2274 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_b1234f77-5c34-4c96-bce0-c457facfb566
|
| 2275 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_83fba088-495e-4423-abe6-b30a70233d4c
|
| 2276 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_oximeter_a0570de6-fbf4-4073-9e84-a6e01264b6a9
|
| 2277 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_3773d36c-8002-400f-9f73-719a40bd0f35
|
| 2278 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_external_dns_1999ec12-789c-40c8-83be-a68bad0fad69
|
| 2279 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_nexus_38b5bd68-0c71-49cc-854d-e90d9972a6f5
|
| 2280 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_clickhouse_b2eaae1a-392a-411d-b8cc-0dff26ddef0c
|
| 2281 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_pantry_7d74b4bc-c408-4088-9473-a933f16aef97
|
| 2282 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_pantry_ad7ed868-02d0-452d-8c54-23c7ef814af4
|
| 2283 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_1b32292f-6190-435a-87d8-7fdbe1a65f21
|
| 2284 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_cad70807-3779-4dbd-b5e8-a3a0e7563ca7
|
| 2285 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_475bc82c-0c96-4f51-8fdf-e66178c8f3fa
|
| 2286 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_external_dns_1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab
|
| 2287 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_nexus_42ed2c6e-2acb-4dbc-9b39-365d99de97d3
|
| 2288 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_pantry_9324d491-c0d2-436b-aafc-efea5edebe21
|
| 2289 | 2024-02-15T00:09:41.606Z | INFO | SledAgent (ServiceManager): Zone started
file = sled-agent/src/services.rs:2766
zone = oxz_crucible_661755cb-0b27-4b29-a851-a3a45274279d
|
| 2290 | 2024-02-15T00:09:41.607Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:198
|
| 2291 | 2024-02-15T00:09:41.609Z | INFO | SledAgent (ServiceManager): Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp
file = common/src/ledger.rs:198
|
| 2292 | 2024-02-15T00:09:41.611Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 74355532
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:56765
req_id = 63c68e8c-a816-4064-ac0c-f9e4459aa3e2
response_code = 204
uri = /omicron-zones
|
| 2293 | 2024-02-15T00:09:41.616Z | INFO | SledAgent (RSS): Finished setting up services
file = sled-agent/src/rack_setup/service.rs:1059
|
| 2294 | 2024-02-15T00:09:41.617Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:198
|
| 2295 | 2024-02-15T00:09:41.617Z | INFO | SledAgent (RSS): Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp
file = common/src/ledger.rs:198
|
| 2296 | 2024-02-15T00:09:41.621Z | INFO | SledAgent (RSS): Handing off control to Nexus
file = sled-agent/src/rack_setup/service.rs:549
|
| 2297 | 2024-02-15T00:09:41.621Z | INFO | SledAgent (RSS): Nexus address: [fd00:1122:3344:101::b]:12221
file = sled-agent/src/rack_setup/service.rs:551
|
| 2298 | 2024-02-15T00:09:41.655Z | INFO | SledAgent (RSS): rack_network_config: RackNetworkConfigV1 {
bgp: [],
infra_ip_first: 192.168.1.30,
infra_ip_last: 192.168.1.30,
ports: [
PortConfigV1 {
addresses: [
V4(
Ipv4Network {
addr: 10.151.2.174,
prefix: 24,
},
),
],
autoneg: false,
bgp_peers: [],
port: "qsfp0",
routes: [
RouteConfig {
destination: V4(
Ipv4Network {
addr: 0.0.0.0,
prefix: 0,
},
),
nexthop: 10.151.2.100,
},
],
switch: Switch0,
uplink_port_fec: None,
uplink_port_speed: Speed40G,
},
],
rack_subnet: Ipv6Network {
addr: fd00:1122:3344:100::,
prefix: 56,
},
}
file = sled-agent/src/rack_setup/service.rs:655
|
| 2299 | 2024-02-15T00:09:41.656Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fdf13508-1e61-48db-a883-3beaba642d42/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:676
|
| 2300 | 2024-02-15T00:09:41.658Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 255.845630749s
|
| 2301 | 2024-02-15T00:09:41.691Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 255.878243868s
|
| 2302 | 2024-02-15T00:09:41.718Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 255.905227206s
|
| 2303 | 2024-02-15T00:09:41.723Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fdf13508-1e61-48db-a883-3beaba642d42/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:676
|
| 2304 | 2024-02-15T00:09:41.789Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 255.975162281s
|
| 2305 | 2024-02-15T00:09:41.811Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fdf13508-1e61-48db-a883-3beaba642d42/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:676
|
| 2306 | 2024-02-15T00:09:41.875Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2307 | 2024-02-15T00:09:41.877Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/sled_agent.rs:741
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
total duration = 245.755577603s
|
| 2308 | 2024-02-15T00:09:41.898Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 256.08694675s
|
| 2309 | 2024-02-15T00:09:41.907Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:223
total duration = 256.09548775s
|
| 2310 | 2024-02-15T00:09:42.004Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c76945c0-20aa-4b87-817a-70fea9ec6d5a/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)"
file = sled-agent/src/storage_monitor.rs:273
total duration = 256.191076715s
|
| 2311 | 2024-02-15T00:09:42.015Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fdf13508-1e61-48db-a883-3beaba642d42/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:676
|
| 2312 | 2024-02-15T00:09:42.140Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fdf13508-1e61-48db-a883-3beaba642d42/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)
file = sled-agent/src/rack_setup/service.rs:676
|
| 2313 | 2024-02-15T00:09:42.346Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "48b066d6-729e-4f5e-9200-29775e06db75", "content-length": "160", "date": "Thu, 15 Feb 2024 00:09:42 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: ip-pool with name \"oxide-service-pool\"", request_id: "48b066d6-729e-4f5e-9200-29775e06db75" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2314 | 2024-02-15T00:09:42.539Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c008e4f4-baf6-4278-a28f-146a5ba01809", "content-length": "124", "date": "Thu, 15 Feb 2024 00:09:42 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c008e4f4-baf6-4278-a28f-146a5ba01809" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2315 | 2024-02-15T00:09:42.762Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3da3b0eb-3e11-4787-b314-7bc1c91a0a8e", "content-length": "124", "date": "Thu, 15 Feb 2024 00:09:42 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3da3b0eb-3e11-4787-b314-7bc1c91a0a8e" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2316 | 2024-02-15T00:09:42.784Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Error Response: status: 404 Not Found; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"488d3754-c349-4e05-81bc-db71ab118b6a\\", \\"content-length\\": \\"272\\", \\"date\\": \\"Thu, 15 Feb 2024 00:09:42 GMT\\"}; value: Error { error_code: Some(\\"ObjectNotFound\\"), message: \\"not found: physical-disk with id \\\\\\"vendor = \\\\\\"synthetic-vendor\\\\\\", serial = \\\\\\"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\", model = \\\\\\"synthetic-model\\\\\\"\\\\\\"\\", request_id: \\"488d3754-c349-4e05-81bc-db71ab118b6a\\" }"
file = sled-agent/src/storage_monitor.rs:273
total duration = 256.970340071s
|
| 2317 | 2024-02-15T00:09:42.819Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Error Response: status: 404 Not Found; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"995a0562-3330-4f9d-ac6d-096b6b955502\\", \\"content-length\\": \\"173\\", \\"date\\": \\"Thu, 15 Feb 2024 00:09:42 GMT\\"}; value: Error { error_code: Some(\\"ObjectNotFound\\"), message: \\"not found: sled with id \\\\\\"c76945c0-20aa-4b87-817a-70fea9ec6d5a\\\\\\"\\", request_id: \\"995a0562-3330-4f9d-ac6d-096b6b955502\\" }"
file = sled-agent/src/storage_monitor.rs:223
total duration = 257.007343548s
|
| 2318 | 2024-02-15T00:09:43.175Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "de1381fe-b1bb-4ff1-8ddb-74e776a987de", "content-length": "124", "date": "Thu, 15 Feb 2024 00:09:42 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "de1381fe-b1bb-4ff1-8ddb-74e776a987de" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2319 | 2024-02-15T00:09:43.193Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Error Response: status: 404 Not Found; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"0cf574e6-82c1-439d-9583-fec3173a086b\\", \\"content-length\\": \\"173\\", \\"date\\": \\"Thu, 15 Feb 2024 00:09:42 GMT\\"}; value: Error { error_code: Some(\\"ObjectNotFound\\"), message: \\"not found: sled with id \\\\\\"c76945c0-20aa-4b87-817a-70fea9ec6d5a\\\\\\"\\", request_id: \\"0cf574e6-82c1-439d-9583-fec3173a086b\\" }"
file = sled-agent/src/storage_monitor.rs:223
total duration = 257.380867296s
|
| 2320 | 2024-02-15T00:09:43.804Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "0b58851e-db9c-4959-902a-de749c4215ea", "content-length": "124", "date": "Thu, 15 Feb 2024 00:09:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "0b58851e-db9c-4959-902a-de749c4215ea" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2321 | 2024-02-15T00:09:43.864Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
err = "Error Response: status: 404 Not Found; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"39037b30-5578-45ce-92d2-481e40fc4207\\", \\"content-length\\": \\"173\\", \\"date\\": \\"Thu, 15 Feb 2024 00:09:43 GMT\\"}; value: Error { error_code: Some(\\"ObjectNotFound\\"), message: \\"not found: sled with id \\\\\\"c76945c0-20aa-4b87-817a-70fea9ec6d5a\\\\\\"\\", request_id: \\"39037b30-5578-45ce-92d2-481e40fc4207\\" }"
file = sled-agent/src/storage_monitor.rs:223
total duration = 258.053982694s
|
| 2322 | 2024-02-15T00:09:44.402Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8d975738-a501-4363-84fb-70a056e1655f", "content-length": "124", "date": "Thu, 15 Feb 2024 00:09:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8d975738-a501-4363-84fb-70a056e1655f" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2323 | 2024-02-15T00:09:44.680Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
sled_id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2324 | 2024-02-15T00:09:44.713Z | INFO | SledAgent (HardwareMonitor): contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "sock" }
file = sled-agent/src/sled_agent.rs:698
id = c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2325 | 2024-02-15T00:09:45.468Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "6023bbef-7419-4a1c-ae82-36a575f73bf8", "content-length": "174", "date": "Thu, 15 Feb 2024 00:09:45 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"14b4dc87-ab46-49fb-a4b4-d361ae214c03\"", request_id: "6023bbef-7419-4a1c-ae82-36a575f73bf8" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2326 | 2024-02-15T00:09:45.794Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", model: "synthetic-model" }
err = "Error Response: status: 404 Not Found; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"7137800e-b526-497c-8bb8-c7bbe823bc8f\\", \\"content-length\\": \\"272\\", \\"date\\": \\"Thu, 15 Feb 2024 00:09:45 GMT\\"}; value: Error { error_code: Some(\\"ObjectNotFound\\"), message: \\"not found: physical-disk with id \\\\\\"vendor = \\\\\\"synthetic-vendor\\\\\\", serial = \\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\", model = \\\\\\"synthetic-model\\\\\\"\\\\\\"\\", request_id: \\"7137800e-b526-497c-8bb8-c7bbe823bc8f\\" }"
file = sled-agent/src/storage_monitor.rs:273
total duration = 259.980867199s
|
| 2327 | 2024-02-15T00:09:46.538Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "06baa4b1-33bf-472f-97e4-da3cef764a6a", "content-length": "174", "date": "Thu, 15 Feb 2024 00:09:46 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"14b4dc87-ab46-49fb-a4b4-d361ae214c03\"", request_id: "06baa4b1-33bf-472f-97e4-da3cef764a6a" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2328 | 2024-02-15T00:09:46.712Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", model: "synthetic-model" }
err = "Error Response: status: 404 Not Found; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"a5cb1ef0-5ae2-4e0a-95b2-c0858082aef1\\", \\"content-length\\": \\"272\\", \\"date\\": \\"Thu, 15 Feb 2024 00:09:46 GMT\\"}; value: Error { error_code: Some(\\"ObjectNotFound\\"), message: \\"not found: physical-disk with id \\\\\\"vendor = \\\\\\"synthetic-vendor\\\\\\", serial = \\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\", model = \\\\\\"synthetic-model\\\\\\"\\\\\\"\\", request_id: \\"a5cb1ef0-5ae2-4e0a-95b2-c0858082aef1\\" }"
file = sled-agent/src/storage_monitor.rs:273
total duration = 260.899158936s
|
| 2329 | 2024-02-15T00:09:47.275Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
err = "Error Response: status: 404 Not Found; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"8290b76d-dce6-42f8-b1fd-eb720cb81574\\", \\"content-length\\": \\"272\\", \\"date\\": \\"Thu, 15 Feb 2024 00:09:46 GMT\\"}; value: Error { error_code: Some(\\"ObjectNotFound\\"), message: \\"not found: physical-disk with id \\\\\\"vendor = \\\\\\"synthetic-vendor\\\\\\", serial = \\\\\\"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\", model = \\\\\\"synthetic-model\\\\\\"\\\\\\"\\", request_id: \\"8290b76d-dce6-42f8-b1fd-eb720cb81574\\" }"
file = sled-agent/src/storage_monitor.rs:273
total duration = 261.46322478s
|
| 2330 | 2024-02-15T00:09:47.448Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "cbeee3b5-8366-4cc9-a661-4dd320b9d4d2", "content-length": "174", "date": "Thu, 15 Feb 2024 00:09:46 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"14b4dc87-ab46-49fb-a4b4-d361ae214c03\"", request_id: "cbeee3b5-8366-4cc9-a661-4dd320b9d4d2" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2331 | 2024-02-15T00:09:48.962Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
err = "Error Response: status: 404 Not Found; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"ee3a10ca-929c-428e-9698-4bbbc3d7c783\\", \\"content-length\\": \\"272\\", \\"date\\": \\"Thu, 15 Feb 2024 00:09:48 GMT\\"}; value: Error { error_code: Some(\\"ObjectNotFound\\"), message: \\"not found: physical-disk with id \\\\\\"vendor = \\\\\\"synthetic-vendor\\\\\\", serial = \\\\\\"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\", model = \\\\\\"synthetic-model\\\\\\"\\\\\\"\\", request_id: \\"ee3a10ca-929c-428e-9698-4bbbc3d7c783\\" }"
file = sled-agent/src/storage_monitor.rs:273
total duration = 263.148862896s
|
| 2332 | 2024-02-15T00:09:49.343Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "f9840426-303d-4536-a414-50265f70e4b6", "content-length": "174", "date": "Thu, 15 Feb 2024 00:09:48 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"14b4dc87-ab46-49fb-a4b4-d361ae214c03\"", request_id: "f9840426-303d-4536-a414-50265f70e4b6" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2333 | 2024-02-15T00:09:49.507Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2334 | 2024-02-15T00:09:49.508Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2335 | 2024-02-15T00:09:49.508Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2336 | 2024-02-15T00:09:50.158Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2337 | 2024-02-15T00:09:50.158Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2338 | 2024-02-15T00:09:50.158Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: M2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2339 | 2024-02-15T00:09:51.007Z | WARN | SledAgent (StorageMonitor): failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", model: "synthetic-model" }
err = "Error Response: status: 404 Not Found; headers: {\\"content-type\\": \\"application/json\\", \\"x-request-id\\": \\"ea1d6b72-6fd8-4d13-83ca-ac668d0b4f2d\\", \\"content-length\\": \\"272\\", \\"date\\": \\"Thu, 15 Feb 2024 00:09:50 GMT\\"}; value: Error { error_code: Some(\\"ObjectNotFound\\"), message: \\"not found: physical-disk with id \\\\\\"vendor = \\\\\\"synthetic-vendor\\\\\\", serial = \\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\", model = \\\\\\"synthetic-model\\\\\\"\\\\\\"\\", request_id: \\"ea1d6b72-6fd8-4d13-83ca-ac668d0b4f2d\\" }"
file = sled-agent/src/storage_monitor.rs:273
total duration = 265.193448957s
|
| 2340 | 2024-02-15T00:09:51.120Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "d026365f-d129-4c39-b6c7-949712a54035", "content-length": "174", "date": "Thu, 15 Feb 2024 00:09:50 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"14b4dc87-ab46-49fb-a4b4-d361ae214c03\"", request_id: "d026365f-d129-4c39-b6c7-949712a54035" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2341 | 2024-02-15T00:09:51.404Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48586
|
| 2342 | 2024-02-15T00:09:51.405Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 45
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:48586
req_id = c6c4b870-989d-474f-8e31-f907101a9a73
response_code = 200
uri = /baseboard
|
| 2343 | 2024-02-15T00:09:52.374Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2344 | 2024-02-15T00:09:52.374Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2345 | 2024-02-15T00:09:52.374Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2346 | 2024-02-15T00:09:52.374Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2347 | 2024-02-15T00:09:52.374Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: Put(PhysicalDiskPutRequest { model: "synthetic-model", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", sled_id: c76945c0-20aa-4b87-817a-70fea9ec6d5a, variant: U2, vendor: "synthetic-vendor" })
file = sled-agent/src/storage_monitor.rs:105
|
| 2348 | 2024-02-15T00:09:52.887Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "38042dd0-767f-4b6d-b6fb-58284bace98c", "content-length": "174", "date": "Thu, 15 Feb 2024 00:09:51 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"14b4dc87-ab46-49fb-a4b4-d361ae214c03\"", request_id: "38042dd0-767f-4b6d-b6fb-58284bace98c" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2349 | 2024-02-15T00:09:53.649Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2350 | 2024-02-15T00:09:53.649Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2351 | 2024-02-15T00:09:53.649Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2352 | 2024-02-15T00:09:53.649Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2353 | 2024-02-15T00:09:53.649Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2354 | 2024-02-15T00:09:54.335Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "5fb5c751-5454-4a3a-9838-ccee04e3c3c3", "content-length": "174", "date": "Thu, 15 Feb 2024 00:09:54 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\"", request_id: "5fb5c751-5454-4a3a-9838-ccee04e3c3c3" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2355 | 2024-02-15T00:09:56.155Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "169a15af-a32d-4aec-b698-ccfc5bbab05b", "content-length": "174", "date": "Thu, 15 Feb 2024 00:09:56 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\"", request_id: "169a15af-a32d-4aec-b698-ccfc5bbab05b" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2356 | 2024-02-15T00:09:57.967Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "a0ebe89b-6679-4017-be0b-96bfd4b2aa7c", "content-length": "174", "date": "Thu, 15 Feb 2024 00:09:57 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\"", request_id: "a0ebe89b-6679-4017-be0b-96bfd4b2aa7c" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2357 | 2024-02-15T00:09:58.877Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2358 | 2024-02-15T00:10:02.498Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "d7b0f5e9-6404-4f29-935e-3a7b07276172", "content-length": "174", "date": "Thu, 15 Feb 2024 00:10:02 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\"", request_id: "d7b0f5e9-6404-4f29-935e-3a7b07276172" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2359 | 2024-02-15T00:10:02.939Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2360 | 2024-02-15T00:10:02.939Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2361 | 2024-02-15T00:10:02.939Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2362 | 2024-02-15T00:10:02.939Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2363 | 2024-02-15T00:10:02.939Z | INFO | SledAgent (StorageMonitor): Nexus notification complete: ZpoolPutRequest { disk_model: "synthetic-model", disk_serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", disk_vendor: "synthetic-vendor", size: ByteCount(20937965568) }
file = sled-agent/src/storage_monitor.rs:105
|
| 2364 | 2024-02-15T00:10:06.568Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:43334
|
| 2365 | 2024-02-15T00:10:06.572Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2366 | 2024-02-15T00:10:06.572Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2367 | 2024-02-15T00:10:06.572Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2368 | 2024-02-15T00:10:06.572Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2369 | 2024-02-15T00:10:06.572Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2370 | 2024-02-15T00:10:06.572Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2371 | 2024-02-15T00:10:06.572Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2372 | 2024-02-15T00:10:06.572Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 4167
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:43334
req_id = d0b8872f-8b27-4319-aee6-df970101cccb
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2373 | 2024-02-15T00:10:06.692Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "7814c8ea-eeee-48a3-9246-d5edd5445256", "content-length": "127", "date": "Thu, 15 Feb 2024 00:10:05 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "7814c8ea-eeee-48a3-9246-d5edd5445256" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2374 | 2024-02-15T00:10:06.818Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:51317
|
| 2375 | 2024-02-15T00:10:06.818Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 512
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:51317
req_id = b52539cf-f857-4a56-b206-928738e5ae0f
response_code = 200
uri = /inventory
|
| 2376 | 2024-02-15T00:10:06.819Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2377 | 2024-02-15T00:10:06.819Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2378 | 2024-02-15T00:10:06.820Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2379 | 2024-02-15T00:10:06.820Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 1564
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:51317
req_id = c53f0300-662b-42a4-b7c8-0bb706e58f5d
response_code = 200
uri = /omicron-zones
|
| 2380 | 2024-02-15T00:10:09.939Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:41520
|
| 2381 | 2024-02-15T00:10:09.939Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2382 | 2024-02-15T00:10:09.939Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2383 | 2024-02-15T00:10:09.939Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2384 | 2024-02-15T00:10:09.939Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2385 | 2024-02-15T00:10:09.939Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2386 | 2024-02-15T00:10:09.939Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2387 | 2024-02-15T00:10:09.939Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2388 | 2024-02-15T00:10:09.939Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 866
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:41520
req_id = 92c242dd-6f18-4d68-bb3d-61bcbf786b8a
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2389 | 2024-02-15T00:10:10.017Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "73ca300f-ef75-49af-91d4-9f7aacbe8d87", "content-length": "127", "date": "Thu, 15 Feb 2024 00:10:09 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "73ca300f-ef75-49af-91d4-9f7aacbe8d87" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2390 | 2024-02-15T00:10:10.100Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:38011
|
| 2391 | 2024-02-15T00:10:10.100Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 44
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:38011
req_id = b26a7c70-b0bf-458e-9d01-5e96891638c3
response_code = 200
uri = /inventory
|
| 2392 | 2024-02-15T00:10:10.100Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2393 | 2024-02-15T00:10:10.100Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2394 | 2024-02-15T00:10:10.101Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2395 | 2024-02-15T00:10:10.101Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 1046
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:38011
req_id = c2dbee7f-c60f-42ae-aac7-d38a7976578d
response_code = 200
uri = /omicron-zones
|
| 2396 | 2024-02-15T00:10:15.725Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:38638
|
| 2397 | 2024-02-15T00:10:15.725Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2398 | 2024-02-15T00:10:15.726Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2399 | 2024-02-15T00:10:15.726Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2400 | 2024-02-15T00:10:15.726Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2401 | 2024-02-15T00:10:15.726Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2402 | 2024-02-15T00:10:15.726Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2403 | 2024-02-15T00:10:15.726Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2404 | 2024-02-15T00:10:15.726Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 369
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:38638
req_id = 08e39ee5-86bd-4542-8327-7f51d6ddec7f
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2405 | 2024-02-15T00:10:15.773Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "6503f5ee-42b5-4ac2-8b42-37568eeda8d1", "content-length": "127", "date": "Thu, 15 Feb 2024 00:10:15 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "6503f5ee-42b5-4ac2-8b42-37568eeda8d1" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2406 | 2024-02-15T00:10:15.848Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:48136
|
| 2407 | 2024-02-15T00:10:15.848Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 47
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:48136
req_id = 1712d548-e861-402b-a954-c7309b4bbef2
response_code = 200
uri = /inventory
|
| 2408 | 2024-02-15T00:10:15.848Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2409 | 2024-02-15T00:10:15.848Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2410 | 2024-02-15T00:10:15.848Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2411 | 2024-02-15T00:10:15.849Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 429
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:48136
req_id = a3f786ad-d1df-4912-98ee-c3afa222fece
response_code = 200
uri = /omicron-zones
|
| 2412 | 2024-02-15T00:10:21.439Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:42907
|
| 2413 | 2024-02-15T00:10:21.440Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 678
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:42907
req_id = 0b76616a-0a29-4f88-9b56-515fc3c8855a
response_code = 200
uri = /baseboard
|
| 2414 | 2024-02-15T00:10:21.575Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:40551
|
| 2415 | 2024-02-15T00:10:21.575Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2416 | 2024-02-15T00:10:21.576Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2417 | 2024-02-15T00:10:21.576Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2418 | 2024-02-15T00:10:21.576Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2419 | 2024-02-15T00:10:21.576Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2420 | 2024-02-15T00:10:21.576Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2421 | 2024-02-15T00:10:21.576Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2422 | 2024-02-15T00:10:21.576Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 359
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:40551
req_id = de22271c-7d98-4ff7-8672-9499529fef03
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2423 | 2024-02-15T00:10:21.628Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "dd08b6a9-23fe-4050-95b8-428958d87a50", "content-length": "127", "date": "Thu, 15 Feb 2024 00:10:21 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "dd08b6a9-23fe-4050-95b8-428958d87a50" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2424 | 2024-02-15T00:10:21.704Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:52485
|
| 2425 | 2024-02-15T00:10:21.704Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 45
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:52485
req_id = 36757c98-eb5f-4e94-9670-b0ce1fdf637d
response_code = 200
uri = /inventory
|
| 2426 | 2024-02-15T00:10:21.704Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2427 | 2024-02-15T00:10:21.704Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2428 | 2024-02-15T00:10:21.704Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2429 | 2024-02-15T00:10:21.705Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 474
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:52485
req_id = f50c8266-8082-480e-8b61-252b8f88a57b
response_code = 200
uri = /omicron-zones
|
| 2430 | 2024-02-15T00:10:23.846Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::d]:45486
|
| 2431 | 2024-02-15T00:10:23.857Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 10809
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = f409ae92-6bd0-40a2-9c73-33ee1cbb5511
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2432 | 2024-02-15T00:10:29.888Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:58744
|
| 2433 | 2024-02-15T00:10:29.889Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2434 | 2024-02-15T00:10:29.889Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2435 | 2024-02-15T00:10:29.889Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2436 | 2024-02-15T00:10:29.889Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2437 | 2024-02-15T00:10:29.889Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2438 | 2024-02-15T00:10:29.889Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2439 | 2024-02-15T00:10:29.889Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2440 | 2024-02-15T00:10:29.889Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 1038
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:58744
req_id = 11d72972-438d-4a40-81f4-e585cc3c0c0b
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2441 | 2024-02-15T00:10:29.941Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "4e6529c1-6367-4b93-8f51-515e79f3c0d9", "content-length": "127", "date": "Thu, 15 Feb 2024 00:10:29 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "4e6529c1-6367-4b93-8f51-515e79f3c0d9" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2442 | 2024-02-15T00:10:30.028Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:62667
|
| 2443 | 2024-02-15T00:10:30.029Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 53
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:62667
req_id = 6a797ca4-de59-487a-864e-d2e8d783053d
response_code = 200
uri = /inventory
|
| 2444 | 2024-02-15T00:10:30.029Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2445 | 2024-02-15T00:10:30.029Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2446 | 2024-02-15T00:10:30.029Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2447 | 2024-02-15T00:10:30.029Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 670
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:62667
req_id = 5ab0e011-284e-4185-a8a1-dc54056c808e
response_code = 200
uri = /omicron-zones
|
| 2448 | 2024-02-15T00:10:34.842Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:38772
|
| 2449 | 2024-02-15T00:10:34.842Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2450 | 2024-02-15T00:10:34.842Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2451 | 2024-02-15T00:10:34.842Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2452 | 2024-02-15T00:10:34.842Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2453 | 2024-02-15T00:10:34.842Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2454 | 2024-02-15T00:10:34.842Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2455 | 2024-02-15T00:10:34.842Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2456 | 2024-02-15T00:10:34.842Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 355
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:38772
req_id = 451b106a-bafd-489b-9153-24c5c868aeb0
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2457 | 2024-02-15T00:10:34.886Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "33cb639e-6323-4b22-9137-d1ae0aaa1723", "content-length": "127", "date": "Thu, 15 Feb 2024 00:10:34 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "33cb639e-6323-4b22-9137-d1ae0aaa1723" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2458 | 2024-02-15T00:10:34.968Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:56065
|
| 2459 | 2024-02-15T00:10:34.968Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 31
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:56065
req_id = 811549ae-a53a-4c4b-a4d2-0206d1599292
response_code = 200
uri = /inventory
|
| 2460 | 2024-02-15T00:10:34.968Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2461 | 2024-02-15T00:10:34.968Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2462 | 2024-02-15T00:10:34.970Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2463 | 2024-02-15T00:10:34.970Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 2012
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:56065
req_id = 9b04e976-02c4-4224-b886-e48538cf9a06
response_code = 200
uri = /omicron-zones
|
| 2464 | 2024-02-15T00:10:50.669Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:42100
|
| 2465 | 2024-02-15T00:10:50.669Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2466 | 2024-02-15T00:10:50.669Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2467 | 2024-02-15T00:10:50.669Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2468 | 2024-02-15T00:10:50.669Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2469 | 2024-02-15T00:10:50.669Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2470 | 2024-02-15T00:10:50.669Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2471 | 2024-02-15T00:10:50.669Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2472 | 2024-02-15T00:10:50.669Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 353
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:42100
req_id = 8dbdcba0-56fb-45d1-9120-b1126e35af98
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2473 | 2024-02-15T00:10:50.722Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "d9957622-f9a2-40d2-9ee7-e4d2e0437e7d", "content-length": "127", "date": "Thu, 15 Feb 2024 00:10:50 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "d9957622-f9a2-40d2-9ee7-e4d2e0437e7d" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2474 | 2024-02-15T00:10:50.808Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:43288
|
| 2475 | 2024-02-15T00:10:50.809Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 599
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:43288
req_id = fa6ebcd8-e57f-47ee-96e1-a7da81a76fdd
response_code = 200
uri = /inventory
|
| 2476 | 2024-02-15T00:10:50.809Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2477 | 2024-02-15T00:10:50.809Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2478 | 2024-02-15T00:10:50.809Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2479 | 2024-02-15T00:10:50.810Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 487
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:43288
req_id = f64e7f4e-6335-4380-8c70-4b6b8daf1222
response_code = 200
uri = /omicron-zones
|
| 2480 | 2024-02-15T00:10:51.472Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47392
|
| 2481 | 2024-02-15T00:10:51.472Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 29
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:47392
req_id = a4bdd733-a263-4a39-8848-b17c8f13d0cd
response_code = 200
uri = /baseboard
|
| 2482 | 2024-02-15T00:10:53.846Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 192
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = cc90d5c9-a2a5-46ce-afc6-0e3f7135bb0a
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2483 | 2024-02-15T00:11:08.414Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:776
|
| 2484 | 2024-02-15T00:11:09.691Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:49113
|
| 2485 | 2024-02-15T00:11:09.691Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2486 | 2024-02-15T00:11:09.691Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2487 | 2024-02-15T00:11:09.691Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2488 | 2024-02-15T00:11:09.691Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2489 | 2024-02-15T00:11:09.691Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2490 | 2024-02-15T00:11:09.691Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2491 | 2024-02-15T00:11:09.691Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2492 | 2024-02-15T00:11:09.691Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 219
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:49113
req_id = eff13b0b-ec06-499a-9224-13b3c608d0e2
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2493 | 2024-02-15T00:11:09.741Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "f11afaaa-2890-41e7-ab31-ee7e7236d52a", "content-length": "127", "date": "Thu, 15 Feb 2024 00:11:09 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "f11afaaa-2890-41e7-ab31-ee7e7236d52a" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2494 | 2024-02-15T00:11:09.836Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:41816
|
| 2495 | 2024-02-15T00:11:09.836Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 38
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:41816
req_id = fbd0ecba-8bd3-40e1-a928-fe218f5eaffb
response_code = 200
uri = /inventory
|
| 2496 | 2024-02-15T00:11:09.836Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2497 | 2024-02-15T00:11:09.836Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2498 | 2024-02-15T00:11:09.836Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2499 | 2024-02-15T00:11:09.837Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 469
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:41816
req_id = c86de967-3b47-4982-8a30-71a6c3ac60d7
response_code = 200
uri = /omicron-zones
|
| 2500 | 2024-02-15T00:11:21.504Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56873
|
| 2501 | 2024-02-15T00:11:21.504Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 36
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56873
req_id = a10f58cb-2bd5-48a8-a1ff-4d1a336a46d0
response_code = 200
uri = /baseboard
|
| 2502 | 2024-02-15T00:11:22.506Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:61567
|
| 2503 | 2024-02-15T00:11:22.506Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2504 | 2024-02-15T00:11:22.506Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2505 | 2024-02-15T00:11:22.506Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2506 | 2024-02-15T00:11:22.506Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2507 | 2024-02-15T00:11:22.506Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2508 | 2024-02-15T00:11:22.506Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2509 | 2024-02-15T00:11:22.506Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2510 | 2024-02-15T00:11:22.506Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 236
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:61567
req_id = 6bbc2d5f-cfdb-431d-894b-bcde2ed634a2
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2511 | 2024-02-15T00:11:22.555Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "ca661aea-e963-4db5-a5c7-e176becfb8ef", "content-length": "127", "date": "Thu, 15 Feb 2024 00:11:22 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "ca661aea-e963-4db5-a5c7-e176becfb8ef" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2512 | 2024-02-15T00:11:22.645Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:52627
|
| 2513 | 2024-02-15T00:11:22.645Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 42
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:52627
req_id = 724baa39-5172-4e89-b65b-a3838dc29114
response_code = 200
uri = /inventory
|
| 2514 | 2024-02-15T00:11:22.645Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2515 | 2024-02-15T00:11:22.645Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2516 | 2024-02-15T00:11:22.646Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2517 | 2024-02-15T00:11:22.646Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 499
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:52627
req_id = 54a735b0-ae72-4bbd-9c6b-21b0c29cb0c9
response_code = 200
uri = /omicron-zones
|
| 2518 | 2024-02-15T00:11:23.846Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 165
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = 68266090-5766-4683-bcf3-73a0f5578067
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2519 | 2024-02-15T00:11:43.042Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:59051
|
| 2520 | 2024-02-15T00:11:43.042Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2521 | 2024-02-15T00:11:43.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2522 | 2024-02-15T00:11:43.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2523 | 2024-02-15T00:11:43.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2524 | 2024-02-15T00:11:43.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2525 | 2024-02-15T00:11:43.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2526 | 2024-02-15T00:11:43.042Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2527 | 2024-02-15T00:11:43.042Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 250
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:59051
req_id = 20aa09c5-0f5c-46c7-8d8d-96f2a30c56d7
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2528 | 2024-02-15T00:11:43.093Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "2eeafcd7-5b60-4118-9c6e-0c64c163aca5", "content-length": "127", "date": "Thu, 15 Feb 2024 00:11:43 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "2eeafcd7-5b60-4118-9c6e-0c64c163aca5" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2529 | 2024-02-15T00:11:43.179Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:45489
|
| 2530 | 2024-02-15T00:11:43.179Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 36
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:45489
req_id = ef80e368-9afb-42c2-96d0-547feff24e43
response_code = 200
uri = /inventory
|
| 2531 | 2024-02-15T00:11:43.180Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2532 | 2024-02-15T00:11:43.180Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2533 | 2024-02-15T00:11:43.180Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2534 | 2024-02-15T00:11:43.180Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 1164
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:45489
req_id = a31ea8a8-50ee-4bf2-8856-8b6682aa5c47
response_code = 200
uri = /omicron-zones
|
| 2535 | 2024-02-15T00:11:51.535Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45189
|
| 2536 | 2024-02-15T00:11:51.535Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 42
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:45189
req_id = 4722b5e1-9233-4d9e-9dc1-9a0b5bf81984
response_code = 200
uri = /baseboard
|
| 2537 | 2024-02-15T00:11:53.847Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 177
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = 644cf725-eac7-40d8-8096-613cdb48190f
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2538 | 2024-02-15T00:11:56.243Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:48924
|
| 2539 | 2024-02-15T00:11:56.243Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2540 | 2024-02-15T00:11:56.243Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2541 | 2024-02-15T00:11:56.243Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2542 | 2024-02-15T00:11:56.243Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2543 | 2024-02-15T00:11:56.243Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2544 | 2024-02-15T00:11:56.243Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2545 | 2024-02-15T00:11:56.243Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2546 | 2024-02-15T00:11:56.243Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 239
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:48924
req_id = 58aafc7a-bb9a-436d-a7a5-5592e548c950
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2547 | 2024-02-15T00:11:56.298Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "e3470340-1d7b-43fd-be5e-94d2a3051960", "content-length": "127", "date": "Thu, 15 Feb 2024 00:11:56 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "e3470340-1d7b-43fd-be5e-94d2a3051960" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2548 | 2024-02-15T00:11:56.407Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:56496
|
| 2549 | 2024-02-15T00:11:56.407Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 40
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:56496
req_id = 101e1843-1622-41b9-8d6f-58874ade649c
response_code = 200
uri = /inventory
|
| 2550 | 2024-02-15T00:11:56.407Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2551 | 2024-02-15T00:11:56.407Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2552 | 2024-02-15T00:11:56.407Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2553 | 2024-02-15T00:11:56.407Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 418
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:56496
req_id = 89295c3b-e1f0-4a51-8ddc-4b63b6e30d6b
response_code = 200
uri = /omicron-zones
|
| 2554 | 2024-02-15T00:12:21.569Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:51843
|
| 2555 | 2024-02-15T00:12:21.569Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 27
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:51843
req_id = 18ded7a9-686c-47d2-b667-d09627ab4631
response_code = 200
uri = /baseboard
|
| 2556 | 2024-02-15T00:12:23.846Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 165
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = d90a0aab-2f94-43d1-9e38-83b3e2406e4c
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2557 | 2024-02-15T00:12:24.766Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:64340
|
| 2558 | 2024-02-15T00:12:24.767Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2559 | 2024-02-15T00:12:24.767Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2560 | 2024-02-15T00:12:24.767Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2561 | 2024-02-15T00:12:24.767Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2562 | 2024-02-15T00:12:24.767Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2563 | 2024-02-15T00:12:24.767Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2564 | 2024-02-15T00:12:24.767Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2565 | 2024-02-15T00:12:24.767Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 236
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:64340
req_id = 62ec6d64-e719-426f-976c-8b3f5c6f4062
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2566 | 2024-02-15T00:12:24.821Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "42a87db8-118f-419c-8924-6d49b805f453", "content-length": "127", "date": "Thu, 15 Feb 2024 00:12:24 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "42a87db8-118f-419c-8924-6d49b805f453" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2567 | 2024-02-15T00:12:24.915Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:49781
|
| 2568 | 2024-02-15T00:12:24.915Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 39
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:49781
req_id = 579a7d8c-1fbf-4c41-8bd2-63552b745be2
response_code = 200
uri = /inventory
|
| 2569 | 2024-02-15T00:12:24.915Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2570 | 2024-02-15T00:12:24.915Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2571 | 2024-02-15T00:12:24.916Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2572 | 2024-02-15T00:12:24.916Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 1048
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:49781
req_id = 36eb5752-4ea5-4c51-8e9b-da0341c77cea
response_code = 200
uri = /omicron-zones
|
| 2573 | 2024-02-15T00:12:51.605Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:65003
|
| 2574 | 2024-02-15T00:12:51.605Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 45
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:65003
req_id = f9cd6362-7475-41ef-8979-30b5a63d8502
response_code = 200
uri = /baseboard
|
| 2575 | 2024-02-15T00:12:53.846Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 185
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = 6fe705dc-d7bb-4e42-85eb-6d520326ce3a
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2576 | 2024-02-15T00:13:15.330Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:52353
|
| 2577 | 2024-02-15T00:13:15.330Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2578 | 2024-02-15T00:13:15.330Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2579 | 2024-02-15T00:13:15.330Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2580 | 2024-02-15T00:13:15.330Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2581 | 2024-02-15T00:13:15.330Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2582 | 2024-02-15T00:13:15.330Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2583 | 2024-02-15T00:13:15.330Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2584 | 2024-02-15T00:13:15.330Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 240
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:52353
req_id = 39cfbcbf-bc00-48f9-b1bd-2ab94ddc7242
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2585 | 2024-02-15T00:13:15.413Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "6a09fdb4-7bdd-4499-b8c4-37db9a5bc45c", "content-length": "127", "date": "Thu, 15 Feb 2024 00:13:15 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "6a09fdb4-7bdd-4499-b8c4-37db9a5bc45c" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2586 | 2024-02-15T00:13:15.717Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:38617
|
| 2587 | 2024-02-15T00:13:15.717Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 46
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:38617
req_id = 78434270-52ba-41dc-93c3-cc0191da5e37
response_code = 200
uri = /inventory
|
| 2588 | 2024-02-15T00:13:15.718Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2589 | 2024-02-15T00:13:15.718Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2590 | 2024-02-15T00:13:15.718Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2591 | 2024-02-15T00:13:15.719Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 1248
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:38617
req_id = 30567f7c-7986-40ee-8926-17929de9290a
response_code = 200
uri = /omicron-zones
|
| 2592 | 2024-02-15T00:13:21.639Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64166
|
| 2593 | 2024-02-15T00:13:21.639Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 30
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:64166
req_id = 9b8f87e8-667e-47d7-9af2-f165d3c2dcb1
response_code = 200
uri = /baseboard
|
| 2594 | 2024-02-15T00:13:23.846Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 188
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = 9f9a270c-4317-47d8-8480-7919404d411f
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2595 | 2024-02-15T00:13:41.350Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:59591
|
| 2596 | 2024-02-15T00:13:41.350Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2597 | 2024-02-15T00:13:41.350Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2598 | 2024-02-15T00:13:41.350Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2599 | 2024-02-15T00:13:41.350Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2600 | 2024-02-15T00:13:41.350Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2601 | 2024-02-15T00:13:41.350Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2602 | 2024-02-15T00:13:41.350Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2603 | 2024-02-15T00:13:41.350Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 238
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:59591
req_id = c3908126-0464-47f0-8016-9528c90c0411
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2604 | 2024-02-15T00:13:41.435Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "e9655029-28c4-463d-83f7-8ed4ffa71bf8", "content-length": "127", "date": "Thu, 15 Feb 2024 00:13:41 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "e9655029-28c4-463d-83f7-8ed4ffa71bf8" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2605 | 2024-02-15T00:13:41.607Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:41544
|
| 2606 | 2024-02-15T00:13:41.607Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 41
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:41544
req_id = 3cf3eda8-46c5-41e7-b6e7-e5695e610cfe
response_code = 200
uri = /inventory
|
| 2607 | 2024-02-15T00:13:41.608Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2608 | 2024-02-15T00:13:41.608Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2609 | 2024-02-15T00:13:41.608Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2610 | 2024-02-15T00:13:41.608Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 416
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:41544
req_id = 12f2577c-3581-4d78-83a5-01da080a97fe
response_code = 200
uri = /omicron-zones
|
| 2611 | 2024-02-15T00:13:51.672Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60711
|
| 2612 | 2024-02-15T00:13:51.672Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 34
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:60711
req_id = 57425991-1ae0-421a-8123-1acbe9cd0bd0
response_code = 200
uri = /baseboard
|
| 2613 | 2024-02-15T00:13:53.846Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 270
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = 7b62d185-2fcb-4308-ab36-b1f524618b6f
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2614 | 2024-02-15T00:14:21.704Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:49133
|
| 2615 | 2024-02-15T00:14:21.704Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 37
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:49133
req_id = 745ecc81-99c5-49b4-8240-896147b6864b
response_code = 200
uri = /baseboard
|
| 2616 | 2024-02-15T00:14:23.847Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 788
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = eb017b1c-99ca-4b0b-bb48-ed42db36acd1
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2617 | 2024-02-15T00:14:33.750Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:37682
|
| 2618 | 2024-02-15T00:14:33.750Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2619 | 2024-02-15T00:14:33.750Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2620 | 2024-02-15T00:14:33.750Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2621 | 2024-02-15T00:14:33.750Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2622 | 2024-02-15T00:14:33.750Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2623 | 2024-02-15T00:14:33.750Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2624 | 2024-02-15T00:14:33.750Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2625 | 2024-02-15T00:14:33.750Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 252
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:37682
req_id = 55acceef-e6fd-4b86-b71a-a527affe2e9c
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2626 | 2024-02-15T00:14:33.803Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "2884537b-6937-4d8a-b69c-724736f9e0f4", "content-length": "127", "date": "Thu, 15 Feb 2024 00:14:33 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "2884537b-6937-4d8a-b69c-724736f9e0f4" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2627 | 2024-02-15T00:14:33.928Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:33382
|
| 2628 | 2024-02-15T00:14:33.929Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 28
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:33382
req_id = dd6cf21f-458b-4ecc-9f8c-3e50f7cc9c4b
response_code = 200
uri = /inventory
|
| 2629 | 2024-02-15T00:14:33.929Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2630 | 2024-02-15T00:14:33.929Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2631 | 2024-02-15T00:14:33.929Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2632 | 2024-02-15T00:14:33.929Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 643
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:33382
req_id = 503a53e4-cd77-44d4-9680-27c567050f2b
response_code = 200
uri = /omicron-zones
|
| 2633 | 2024-02-15T00:14:51.736Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56445
|
| 2634 | 2024-02-15T00:14:51.736Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 30
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:56445
req_id = 471d6912-71af-4569-9b4b-4d413f2680af
response_code = 200
uri = /baseboard
|
| 2635 | 2024-02-15T00:14:53.846Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 179
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = a21fac27-af64-4342-b6b0-82e54062bd23
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2636 | 2024-02-15T00:15:21.769Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:34427
|
| 2637 | 2024-02-15T00:15:21.769Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 28
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:34427
req_id = 796f8c45-2801-46f3-8e44-be0a35a7bc00
response_code = 200
uri = /baseboard
|
| 2638 | 2024-02-15T00:15:23.294Z | INFO | SledAgent (auto-cleanup-task): running automatic periodic zone bundle cleanup
file = sled-agent/src/zone_bundle.rs:299
|
| 2639 | 2024-02-15T00:15:23.294Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2640 | 2024-02-15T00:15:23.333Z | INFO | SledAgent (auto-cleanup-task): top of bundle cleanup loop
file = sled-agent/src/zone_bundle.rs:288
next_cleanup = Instant { tv_sec: 2289, tv_nsec: 772430782 }
time_to_next_cleanup = 599.999997s
|
| 2641 | 2024-02-15T00:15:23.855Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8927
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = f2c672c0-eb4f-4fe0-8b02-19b8aaf4ad9b
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2642 | 2024-02-15T00:15:29.985Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:56104
|
| 2643 | 2024-02-15T00:15:29.986Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2644 | 2024-02-15T00:15:29.986Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2645 | 2024-02-15T00:15:29.986Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2646 | 2024-02-15T00:15:29.986Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2647 | 2024-02-15T00:15:29.986Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2648 | 2024-02-15T00:15:29.986Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2649 | 2024-02-15T00:15:29.986Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2650 | 2024-02-15T00:15:29.986Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 281
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:56104
req_id = ea15ce36-0b3d-4e65-bff1-a75469890891
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2651 | 2024-02-15T00:15:30.059Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "9f658ad5-42fa-469e-b48d-b07f39e82aac", "content-length": "127", "date": "Thu, 15 Feb 2024 00:15:29 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "9f658ad5-42fa-469e-b48d-b07f39e82aac" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2652 | 2024-02-15T00:15:30.188Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:36322
|
| 2653 | 2024-02-15T00:15:30.188Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 553
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:36322
req_id = edc207f0-4967-46b5-ab1b-ce3227a40a28
response_code = 200
uri = /inventory
|
| 2654 | 2024-02-15T00:15:30.189Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2655 | 2024-02-15T00:15:30.189Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2656 | 2024-02-15T00:15:30.189Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2657 | 2024-02-15T00:15:30.189Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 608
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:36322
req_id = 674e837d-880f-4f0f-8e66-e9c23f60f432
response_code = 200
uri = /omicron-zones
|
| 2658 | 2024-02-15T00:15:51.800Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:43473
|
| 2659 | 2024-02-15T00:15:51.800Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 51
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:43473
req_id = e9d07999-c1e9-4570-b329-f4717d9242c3
response_code = 200
uri = /baseboard
|
| 2660 | 2024-02-15T00:15:53.847Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 954
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = 0eee740e-4b2f-468c-b55e-968df8468626
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2661 | 2024-02-15T00:16:08.415Z | INFO | SledAgent (DumpSetup-worker): No archival destination for crash dumps yet chosen.
file = sled-agent/src/dump_setup.rs:776
|
| 2662 | 2024-02-15T00:16:21.832Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:53020
|
| 2663 | 2024-02-15T00:16:21.832Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 44
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:53020
req_id = f336d330-1fc0-4e6f-ba4d-ed13475e4a1c
response_code = 200
uri = /baseboard
|
| 2664 | 2024-02-15T00:16:23.846Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 186
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = 1db11127-c951-4b40-a432-e256cc7e8731
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2665 | 2024-02-15T00:16:29.298Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:51411
|
| 2666 | 2024-02-15T00:16:29.303Z | INFO | SledAgent (InstanceManager): ensuring instance is registered
file = sled-agent/src/instance_manager.rs:228
hardware = InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name("net0"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name("net0"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name("net0"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: Some([Vpc(Vni(16654092))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name("net0"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-9c6380e3d07", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, read_only_parent: Some(Volume { block_size: 512, id: ad72fa73-db1d-4bda-86fd-e923832de49e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a89791d0-9b8a-48e3-b873-25fdd54a89e0, key: Some("CZUh9G+urVG1RRyq9KaZu+1vHje9xKhTjAM/2HEu5Bo="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::14]:19001", "[fd00:1122:3344:101::12]:19001", "[fd00:1122:3344:101::17]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, key: Some("VyAhAqjFfYr4vKeuRVRSMV66ERIy+JKPSvyW9BigcBo="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::19]:19001", "[fd00:1122:3344:101::15]:19000", "[fd00:1122:3344:101::13]:19000"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB+DgJPWE9YAAAOAk9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAfg4CT1hPWAAADgJPWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5NGZjY2I0MS0xNmEwLTRkZjAtYmNlNi1hMTdiMTRkNGJlMDUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRTFsZU1nNTdmUWlRZGgzRi81Qm14T0h3NUhPWXIzeGlVSVVuVHhkMGc2NyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
instance_runtime = InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }
propolis_addr = [fd00:1122:3344:101::1:0]:12400
propolis_id = d13d05bd-eed0-4965-8d7c-dde7b491d43e
vmm_runtime = VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-15T00:16:28.903818Z }
|
| 2667 | 2024-02-15T00:16:29.303Z | INFO | SledAgent (InstanceManager): registering new instance
file = sled-agent/src/instance_manager.rs:262
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2668 | 2024-02-15T00:16:29.303Z | INFO | SledAgent (InstanceManager): initializing new Instance
file = sled-agent/src/instance.rs:706
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
propolis_id = d13d05bd-eed0-4965-8d7c-dde7b491d43e
state = InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname("localshark") }, nics: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name("net0"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name("net0"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name("net0"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: Some([Vpc(Vni(16654092))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba, kind: Instance { id: 94fccb41-16a0-4df0-bce6-a17b14d4be05 }, name: Name("net0"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 254, 110, 222])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(16654092), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: "nvme", name: "disk-9c6380e3d07", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, read_only_parent: Some(Volume { block_size: 512, id: ad72fa73-db1d-4bda-86fd-e923832de49e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a89791d0-9b8a-48e3-b873-25fdd54a89e0, key: Some("CZUh9G+urVG1RRyq9KaZu+1vHje9xKhTjAM/2HEu5Bo="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::14]:19001", "[fd00:1122:3344:101::12]:19001", "[fd00:1122:3344:101::17]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, key: Some("VyAhAqjFfYr4vKeuRVRSMV66ERIy+JKPSvyW9BigcBo="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::19]:19001", "[fd00:1122:3344:101::15]:19000", "[fd00:1122:3344:101::13]:19000"] } }] } }], cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB+DgJPWE9YAAAOAk9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAfg4CT1hPWAAADgJPWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5NGZjY2I0MS0xNmEwLTRkZjAtYmNlNi1hMTdiMTRkNGJlMDUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRTFsZU1nNTdmUWlRZGgzRi81Qm14T0h3NUhPWXIzeGlVSVVuVHhkMGc2NyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-15T00:16:28.903818Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }
|
| 2669 | 2024-02-15T00:16:29.304Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 5754
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:51411
req_id = 3caa88e2-50d7-45e8-94a1-68b6962215de
response_code = 200
uri = /instances/94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2670 | 2024-02-15T00:16:29.419Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:50453
|
| 2671 | 2024-02-15T00:16:29.441Z | INFO | SledAgent (PortManager): Created OPTE port
file = illumos-utils/src/opte/port_manager.rs:394
port = Port { inner: PortInner { name: "opte6", _ip: 172.30.0.6, mac: MacAddr6([168, 64, 37, 254, 110, 222]), slot: 1, vni: Vni { inner: 16654092 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: "vopte6" } }
|
| 2672 | 2024-02-15T00:16:29.441Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2673 | 2024-02-15T00:16:29.477Z | INFO | SledAgent (InstanceManager): Configuring new Omicron zone: oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
file = illumos-utils/src/zone.rs:315
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2674 | 2024-02-15T00:16:29.499Z | INFO | SledAgent (InstanceManager): Installing Omicron zone: oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
file = illumos-utils/src/zone.rs:354
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2675 | 2024-02-15T00:16:31.567Z | INFO | SledAgent (InstanceManager): Profile for oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="system/illumos/propolis-server">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlInstance0"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::1:0"/>
<propval type="astring" name="listen_port" value="12400"/>
<propval type="astring" name="metric_addr" value="[fd00:1122:3344:101::c]:12221"/>
</property_group>
</instance>
</service>
</service_bundle>
file = sled-agent/src/profile.rs:34
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2676 | 2024-02-15T00:16:31.568Z | INFO | SledAgent (InstanceManager): Zone booting
file = illumos-utils/src/running_zone.rs:513
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
zone = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2677 | 2024-02-15T00:16:31.761Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e") failed: Property not found. retry in 60.83158ms
file = illumos-utils/src/svc.rs:36
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
zone = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2678 | 2024-02-15T00:16:31.834Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e") failed: Property not found. retry in 148.373225ms
file = illumos-utils/src/svc.rs:36
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
zone = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2679 | 2024-02-15T00:16:32.005Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e") failed: Property not found. retry in 111.975713ms
file = illumos-utils/src/svc.rs:36
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
zone = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2680 | 2024-02-15T00:16:32.130Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e") failed: Property not found. retry in 525.554745ms
file = illumos-utils/src/svc.rs:36
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
zone = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2681 | 2024-02-15T00:16:32.666Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e") failed: Property not found. retry in 1.092930954s
file = illumos-utils/src/svc.rs:36
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
zone = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2682 | 2024-02-15T00:16:33.771Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e") failed: Property not found. retry in 644.661587ms
file = illumos-utils/src/svc.rs:36
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
zone = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2683 | 2024-02-15T00:16:34.426Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e") failed: Property not found. retry in 658.38205ms
file = illumos-utils/src/svc.rs:36
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
zone = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2684 | 2024-02-15T00:16:35.093Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e") failed: Property not found. retry in 975.423708ms
file = illumos-utils/src/svc.rs:36
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
zone = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2685 | 2024-02-15T00:16:36.078Z | WARN | SledAgent (InstanceManager): wait for service Some("oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e") failed: Property not found. retry in 1.321263064s
file = illumos-utils/src/svc.rs:36
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
zone = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2686 | 2024-02-15T00:16:37.427Z | INFO | SledAgent (InstanceManager): Started propolis in zone: oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
file = sled-agent/src/instance.rs:1098
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2687 | 2024-02-15T00:16:37.435Z | INFO | SledAgent (InstanceManager): Propolis SMF service is online
file = sled-agent/src/instance.rs:1107
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2688 | 2024-02-15T00:16:37.469Z | INFO | SledAgent (InstanceManager): Propolis HTTP server online
file = sled-agent/src/instance.rs:1121
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2689 | 2024-02-15T00:16:37.470Z | INFO | SledAgent (InstanceManager): Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some("6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB+DgJPWE9YAAAOAk9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAfg4CT1hPWAAADgJPWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5NGZjY2I0MS0xNmEwLTRkZjAtYmNlNi1hMTdiMTRkNGJlMDUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRTFsZU1nNTdmUWlRZGgzRi81Qm14T0h3NUhPWXIzeGlVSVVuVHhkMGc2NyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"), disks: [DiskRequest { device: "nvme", name: "disk-9c6380e3d07", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, read_only_parent: Some(Volume { block_size: 512, id: ad72fa73-db1d-4bda-86fd-e923832de49e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a89791d0-9b8a-48e3-b873-25fdd54a89e0, key: Some("CZUh9G+urVG1RRyq9KaZu+1vHje9xKhTjAM/2HEu5Bo="), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: ["[fd00:1122:3344:101::14]:19001", "[fd00:1122:3344:101::12]:19001", "[fd00:1122:3344:101::17]:19001"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f0b64806-9772-4e8b-bb4d-bb3be49281d6, key: Some("VyAhAqjFfYr4vKeuRVRSMV66ERIy+JKPSvyW9BigcBo="), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: ["[fd00:1122:3344:101::19]:19001", "[fd00:1122:3344:101::15]:19000", "[fd00:1122:3344:101::13]:19000"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: "vopte6", slot: Slot(1) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: "Test description", id: 94fccb41-16a0-4df0-bce6-a17b14d4be05, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, name: "localshark", vcpus: 2 } }
file = sled-agent/src/instance.rs:453
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2690 | 2024-02-15T00:16:37.621Z | INFO | SledAgent (InstanceManager): result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })
file = sled-agent/src/instance.rs:455
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2691 | 2024-02-15T00:16:37.622Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-02-15T00:16:37.622201717Z }
file = sled-agent/src/instance.rs:337
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2692 | 2024-02-15T00:16:37.622Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:359
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
new_instance_state = InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }
new_vmm_state = VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-15T00:16:37.622201717Z }
propolis_id = d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2693 | 2024-02-15T00:16:37.622Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 8203163
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:50453
req_id = e7bec46f-2aa9-4166-80c8-bbc7f38812d6
response_code = 200
uri = /instances/94fccb41-16a0-4df0-bce6-a17b14d4be05/state
|
| 2694 | 2024-02-15T00:16:37.622Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:260
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }, propolis_id: d13d05bd-eed0-4965-8d7c-dde7b491d43e, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-15T00:16:37.622201717Z } }
|
| 2695 | 2024-02-15T00:16:37.675Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-02-15T00:16:37.675623191Z }
file = sled-agent/src/instance.rs:337
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2696 | 2024-02-15T00:16:37.675Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:359
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
new_instance_state = InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }
new_vmm_state = VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-15T00:16:37.675623191Z }
propolis_id = d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2697 | 2024-02-15T00:16:37.675Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:260
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }, propolis_id: d13d05bd-eed0-4965-8d7c-dde7b491d43e, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-15T00:16:37.675623191Z } }
|
| 2698 | 2024-02-15T00:16:45.577Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:52794
|
| 2699 | 2024-02-15T00:16:45.577Z | INFO | SledAgent (PortManager): Ensuring VPC firewall rules
file = illumos-utils/src/opte/port_manager.rs:525
rules = [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07093e9f-ec90-4e45-a414-e7fcc76e8e09, kind: Service { id: 1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab }, name: Name("external-dns-1db31f1b-e3f9-47c5-a2c6-e5eba50ff1ab"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4c8f21bb-b472-483b-9019-9753bc87f89a, kind: Service { id: 1999ec12-789c-40c8-83be-a68bad0fad69 }, name: Name("external-dns-1999ec12-789c-40c8-83be-a68bad0fad69"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1548b081-af5c-4e63-ac5b-af4c8110dd7f, kind: Service { id: 42ed2c6e-2acb-4dbc-9b39-365d99de97d3 }, name: Name("nexus-42ed2c6e-2acb-4dbc-9b39-365d99de97d3"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 144])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 54a64c4d-df4a-41d6-9256-a85e2b2653f9, kind: Service { id: 052b5be3-98fd-4823-9614-96d5055e333c }, name: Name("nexus-052b5be3-98fd-4823-9614-96d5055e333c"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 77d9f57c-a2c8-491c-acb5-51dd22e47d3e, kind: Service { id: 38b5bd68-0c71-49cc-854d-e90d9972a6f5 }, name: Name("nexus-38b5bd68-0c71-49cc-854d-e90d9972a6f5"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]
vni = Vni(100)
|
| 2700 | 2024-02-15T00:16:45.577Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte5"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2701 | 2024-02-15T00:16:45.577Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte4"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2702 | 2024-02-15T00:16:45.577Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte0"
rules = []
|
| 2703 | 2024-02-15T00:16:45.577Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte2"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]
|
| 2704 | 2024-02-15T00:16:45.577Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte1"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2705 | 2024-02-15T00:16:45.577Z | INFO | SledAgent (PortManager): Setting OPTE firewall rules
file = illumos-utils/src/opte/port_manager.rs:544
port = "opte3"
rules = [FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]
|
| 2706 | 2024-02-15T00:16:45.577Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 256
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::b]:52794
req_id = 2b8143f3-306a-4f05-8eaf-17d0c6fdcbc4
response_code = 204
uri = /vpc/001de000-074c-4000-8000-000000000000/firewall/rules
|
| 2707 | 2024-02-15T00:16:45.630Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "d91b6119-1ea2-46e7-9d0c-ec097b7bcb36", "content-length": "127", "date": "Thu, 15 Feb 2024 00:16:45 GMT"}; value: Error { error_code: Some("InvalidRequest"), message: "address not in lot", request_id: "d91b6119-1ea2-46e7-9d0c-ec097b7bcb36" }
file = sled-agent/src/rack_setup/service.rs:676
|
| 2708 | 2024-02-15T00:16:45.725Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:41255
|
| 2709 | 2024-02-15T00:16:45.725Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 67
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:41255
req_id = 886b30b3-d4d4-408d-ad08-fcbeb9b146fc
response_code = 200
uri = /inventory
|
| 2710 | 2024-02-15T00:16:45.726Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2711 | 2024-02-15T00:16:45.726Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2712 | 2024-02-15T00:16:45.726Z | INFO | SledAgent (ServiceManager): Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json
file = common/src/ledger.rs:176
|
| 2713 | 2024-02-15T00:16:45.726Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 1223
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::b]:41255
req_id = 45dec3c3-1111-400f-a7f9-8b8b2adf4836
response_code = 200
uri = /omicron-zones
|
| 2714 | 2024-02-15T00:16:51.866Z | INFO | SledAgent (dropshot (BootstrapAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
remote_addr = [fdb0:18c0:4d0d:a02a::2]:38346
|
| 2715 | 2024-02-15T00:16:51.867Z | INFO | SledAgent (dropshot (BootstrapAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 569
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
method = GET
remote_addr = [fdb0:18c0:4d0d:a02a::2]:38346
req_id = c3da4b6a-e580-473d-a2e4-c8c08bbf7e96
response_code = 200
uri = /baseboard
|
| 2716 | 2024-02-15T00:16:53.847Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 280
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::d]:45486
req_id = 6778ad12-d6a1-441d-a776-bf1b251a0fa7
response_code = 200
uri = /metrics/collect/c76945c0-20aa-4b87-817a-70fea9ec6d5a
|
| 2717 | 2024-02-15T00:16:54.573Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::a]:40058
|
| 2718 | 2024-02-15T00:16:54.574Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 400
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::a]:40058
req_id = 0e3dee90-f096-4434-8926-6f8c5c0c5e83
response_code = 200
uri = /instances/94fccb41-16a0-4df0-bce6-a17b14d4be05/state
|
| 2719 | 2024-02-15T00:16:54.574Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-02-15T00:16:54.574309394Z }
file = sled-agent/src/instance.rs:337
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2720 | 2024-02-15T00:16:54.574Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:359
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
new_instance_state = InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }
new_vmm_state = VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-15T00:16:54.574309394Z }
propolis_id = d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2721 | 2024-02-15T00:16:54.574Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:260
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }, propolis_id: d13d05bd-eed0-4965-8d7c-dde7b491d43e, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-15T00:16:54.574309394Z } }
|
| 2722 | 2024-02-15T00:16:54.593Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopped), migration_status: NoMigration, time: 2024-02-15T00:16:54.593941555Z }
file = sled-agent/src/instance.rs:337
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2723 | 2024-02-15T00:16:54.593Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:359
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
new_instance_state = InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }
new_vmm_state = VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-15T00:16:54.593941555Z }
propolis_id = d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2724 | 2024-02-15T00:16:54.593Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:260
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d13d05bd-eed0-4965-8d7c-dde7b491d43e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T00:16:28.939278274Z }, propolis_id: d13d05bd-eed0-4965-8d7c-dde7b491d43e, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-15T00:16:54.593941555Z } }
|
| 2725 | 2024-02-15T00:16:54.756Z | INFO | SledAgent (InstanceManager): Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-02-15T00:16:54.756784642Z }
file = sled-agent/src/instance.rs:337
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2726 | 2024-02-15T00:16:54.756Z | INFO | SledAgent (InstanceManager): updated state after observing Propolis state change
file = sled-agent/src/instance.rs:359
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
new_instance_state = InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-15T00:16:54.756784642Z }
new_vmm_state = VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-02-15T00:16:54.756784642Z }
propolis_id = d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2727 | 2024-02-15T00:16:54.756Z | INFO | SledAgent (InstanceManager): terminating VMM that has exited
file = sled-agent/src/instance.rs:374
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2728 | 2024-02-15T00:16:54.756Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2729 | 2024-02-15T00:16:54.756Z | INFO | SledAgent (StorageManager): Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })
file = sled-storage/src/manager.rs:346
|
| 2730 | 2024-02-15T00:16:54.757Z | INFO | SledAgent (ZoneBundler): creating zone bundle
context = ZoneBundleContext { storage_dirs: ["/pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone", "/pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone"], cause: TerminatedInstance, extra_log_dirs: ["/pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug", "/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug", "/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/debug", "/pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/debug", "/pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/debug", "/pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/debug", "/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/debug", "/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug", "/pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug"] }
file = sled-agent/src/zone_bundle.rs:436
zone_name = oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
|
| 2731 | 2024-02-15T00:16:55.341Z | INFO | SledAgent (ZoneBundler): finished zone bundle
file = sled-agent/src/zone_bundle.rs:1229
metadata = ZoneBundleMetadata { id: ZoneBundleId { zone_name: "oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e", bundle_id: 2c3db916-2f68-46c5-aef6-05abe9267bc0 }, time_created: 2024-02-15T00:16:54.757895464Z, version: 0, cause: TerminatedInstance }
|
| 2732 | 2024-02-15T00:16:55.341Z | WARN | SledAgent (InstanceManager): Halting and removing zone: oxz_propolis-server_d13d05bd-eed0-4965-8d7c-dde7b491d43e
file = sled-agent/src/instance.rs:547
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2733 | 2024-02-15T00:16:56.912Z | INFO | SledAgent (InstanceManager): halt_and_remove_logged: Previous zone state: Running
file = illumos-utils/src/zone.rs:269
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|
| 2734 | 2024-02-15T00:16:56.938Z | INFO | SledAgent (InstanceManager): Publishing instance state update to Nexus
file = sled-agent/src/instance.rs:260
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
state = SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-15T00:16:54.756784642Z }, propolis_id: d13d05bd-eed0-4965-8d7c-dde7b491d43e, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-02-15T00:16:54.756784642Z } }
|
| 2735 | 2024-02-15T00:16:57.012Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::b]:40009
|
| 2736 | 2024-02-15T00:16:57.013Z | WARN | SledAgent (PortManager): unset_virtual_nic_host unimplmented
file = illumos-utils/src/opte/port_manager.rs:620
|
| 2737 | 2024-02-15T00:16:57.013Z | INFO | SledAgent (dropshot (SledAgent)): request completed
file = /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849
latency_us = 835
local_addr = [fd00:1122:3344:101::1]:12345
method = DELETE
remote_addr = [fd00:1122:3344:101::b]:40009
req_id = e4e03acd-816e-4067-b072-7625488be599
response_code = 204
uri = /v2p/82f06b8b-3f39-4bd0-810f-60b9a9a0a3ba
|
| 2738 | 2024-02-15T00:16:57.108Z | INFO | SledAgent (InstanceManager): State monitoring task complete
file = sled-agent/src/instance.rs:490
instance_id = 94fccb41-16a0-4df0-bce6-a17b14d4be05
|