| 1 | | [ May 24 20:18:13 Enabled. ] |
| 2 | | [ May 24 20:18:13 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] |
| 3 | | [ May 24 20:18:13 Method "start" exited with status 0. ] |
| 4 | | [ May 24 20:18:13 Rereading configuration. ] |
| 5 | | [ May 24 20:18:13 No 'refresh' method defined. Treating as :true. ] |
| 6 | | note: configured to log to "/dev/stdout" |
| 7 | 2023-05-24T20:18:14.042Z | INFO | SledAgent: Starting mg-ddm service
|
| 8 | 2023-05-24T20:18:14.042Z | INFO | SledAgent: Importing mg-ddm service
path = /opt/oxide/mg-ddm/pkg/ddm/manifest.xml
|
| 9 | 2023-05-24T20:18:14.160Z | INFO | SledAgent: Setting mg-ddm interfaces
interfaces = ("net0/ll" "net1/ll")
|
| 10 | 2023-05-24T20:18:14.169Z | INFO | SledAgent: Enabling mg-ddm service
|
| 11 | 2023-05-24T20:18:14.177Z | INFO | SledAgent: detecting (real or simulated) SP
|
| 12 | 2023-05-24T20:18:14.177Z | INFO | SledAgent: setting up bootstrap agent server
|
| 13 | 2023-05-24T20:18:14.686Z | INFO | SledAgent: Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
prefix = Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }
|
| 14 | 2023-05-24T20:18:14.821Z | INFO | SledAgent: Bootstrap Agent monitoring for hardware
|
| 15 | 2023-05-24T20:18:14.859Z | INFO | SledAgent (BootstrapAgent): Starting hardware monitor
sled_mode = Scrimlet { asic: SoftNpu }
|
| 16 | 2023-05-24T20:18:14.859Z | INFO | SledAgent (HardwareManager): Creating HardwareManager
|
| 17 | 2023-05-24T20:18:14.862Z | INFO | SledAgent (HardwareManager): Generated i86pc baseboard Pc { identifier: "393b90b4de3e437ead4e4684e3451fb2", model: "i86pc" }
|
| 18 | 2023-05-24T20:18:14.862Z | WARN | SledAgent (HardwareManager): Device is not a Gimlet (i86pc), proceeding with null hardware view
|
| 19 | 2023-05-24T20:18:14.862Z | INFO | SledAgent (BootstrapAgent): Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 20 | 2023-05-24T20:18:14.862Z | INFO | SledAgent (BootstrapAgent): Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 21 | 2023-05-24T20:18:14.862Z | INFO | SledAgent (BootstrapAgent): Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 22 | 2023-05-24T20:18:14.862Z | INFO | SledAgent (BootstrapAgent): Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 23 | 2023-05-24T20:18:14.862Z | INFO | SledAgent (BootstrapAgent): Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 24 | 2023-05-24T20:18:14.862Z | INFO | SledAgent (StorageManager): Upserting synthetic disk for: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }
|
| 25 | 2023-05-24T20:18:14.867Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
|
| 26 | 2023-05-24T20:18:14.867Z | INFO | SledAgent (BootstrapAgent): Performing full hardware scan
|
| 27 | 2023-05-24T20:18:14.867Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
|
| 28 | 2023-05-24T20:18:14.867Z | INFO | SledAgent (ServiceManager): Enabling switch zone (new)
|
| 29 | 2023-05-24T20:18:14.995Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b",
size: 10200547328,
allocated: 402432,
free: 10200144896,
health: Online,
}
|
| 30 | 2023-05-24T20:18:14.995Z | INFO | SledAgent (StorageManager): Upserting synthetic disk for: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }
|
| 31 | 2023-05-24T20:18:14.995Z | INFO | SledAgent (BootstrapAgent): Found boot disk M.2: (DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal })
|
| 32 | 2023-05-24T20:18:15.009Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_switch
|
| 33 | 2023-05-24T20:18:15.027Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_switch
|
| 34 | 2023-05-24T20:18:15.139Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b",
size: 10200547328,
allocated: 396288,
free: 10200151040,
health: Online,
}
|
| 35 | 2023-05-24T20:18:15.139Z | INFO | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
|
| 36 | 2023-05-24T20:18:15.139Z | INFO | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
|
| 37 | 2023-05-24T20:18:15.139Z | INFO | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
|
| 38 | 2023-05-24T20:18:15.139Z | INFO | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
|
| 39 | 2023-05-24T20:18:15.139Z | INFO | SledAgent (StorageManager): Upserting synthetic disk for: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }
|
| 40 | 2023-05-24T20:18:15.142Z | INFO | SledAgent (dropshot (BootstrapAgent)): listening
local_addr = [fdb0:18c0:4d0d:a02a::1]:80
|
| 41 | 2023-05-24T20:18:15.142Z | INFO | SledAgent (sprockets (BootstrapAgent)): Started listening
local_addr = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 42 | 2023-05-24T20:18:15.144Z | INFO | SledAgent (RSS): Injecting RSS configuration: RackInitializeRequest {
rack_subnet: fd00:1122:3344:100::,
bootstrap_discovery: OnlyOurs,
rack_secret_threshold: 1,
ntp_servers: [
"ntp.eng.oxide.computer",
],
dns_servers: [
"1.1.1.1",
"9.9.9.9",
],
internal_services_ip_pool_ranges: [
V4(
Ipv4Range {
first: 192.168.1.20,
last: 192.168.1.22,
},
),
],
external_dns_zone_name: "oxide.test",
external_certificates: [
Certificate {
cert: [
45,
45,
45,
45,
45,
66,
69,
71,
73,
78,
32,
67,
69,
82,
84,
73,
70,
73,
67,
65,
84,
69,
45,
45,
45,
45,
45,
10,
77,
73,
73,
70,
66,
106,
67,
67,
65,
117,
54,
103,
65,
119,
73,
66,
65,
103,
73,
85,
85,
77,
99,
47,
81,
72,
76,
83,
55,
48,
103,
69,
99,
75,
76,
97,
112,
108,
67,
79,
98,
87,
101,
104,
88,
74,
73,
119,
68,
81,
89,
74,
75,
111,
90,
73,
104,
118,
99,
78,
65,
81,
69,
76,
10,
66,
81,
65,
119,
73,
106,
69,
103,
77,
66,
52,
71,
65,
49,
85,
69,
65,
119,
119,
88,
99,
109,
86,
106,
98,
51,
90,
108,
99,
110,
107,
117,
99,
51,
108,
122,
76,
109,
57,
52,
97,
87,
82,
108,
76,
110,
82,
108,
99,
51,
81,
119,
72,
104,
99,
78,
77,
106,
77,
119,
78,
84,
73,
48,
10,
77,
106,
65,
120,
79,
68,
69,
122,
87,
104,
99,
78,
77,
106,
77,
119,
78,
84,
73,
51,
77,
106,
65,
120,
79,
68,
69,
122,
87,
106,
65,
105,
77,
83,
65,
119,
72,
103,
89,
68,
86,
81,
81,
68,
68,
66,
100,
121,
90,
87,
78,
118,
100,
109,
86,
121,
101,
83,
53,
122,
101,
88,
77,
117,
10,
98,
51,
104,
112,
90,
71,
85,
117,
100,
71,
86,
122,
100,
68,
67,
67,
65,
105,
73,
119,
68,
81,
89,
74,
75,
111,
90,
73,
104,
118,
99,
78,
65,
81,
69,
66,
66,
81,
65,
68,
103,
103,
73,
80,
65,
68,
67,
67,
65,
103,
111,
67,
103,
103,
73,
66,
65,
77,
73,
116,
82,
81,
104,
74,
10,
101,
67,
77,
89,
76,
85,
108,
88,
43,
89,
57,
109,
78,
74,
117,
47,
119,
115,
83,
100,
111,
49,
114,
106,
88,
68,
51,
70,
53,
47,
79,
43,
76,
67,
73,
109,
85,
57,
89,
121,
75,
115,
81,
77,
104,
54,
56,
75,
70,
83,
72,
54,
121,
69,
76,
99,
122,
120,
88,
72,
108,
119,
101,
50,
10,
53,
52,
50,
110,
103,
107,
112,
80,
84,
72,
70,
90,
50,
47,
50,
55,
74,
90,
89,
101,
81,
109,
111,
110,
51,
79,
75,
43,
107,
98,
122,
79,
55,
102,
79,
83,
107,
49,
54,
113,
112,
73,
55,
79,
121,
72,
72,
66,
122,
112,
88,
48,
52,
110,
51,
53,
74,
83,
83,
113,
52,
114,
97,
68,
10,
80,
114,
114,
81,
83,
55,
109,
52,
109,
87,
112,
112,
113,
87,
87,
107,
69,
68,
49,
52,
47,
49,
75,
114,
109,
116,
47,
87,
102,
55,
88,
99,
109,
53,
57,
54,
85,
117,
79,
53,
77,
65,
50,
105,
47,
48,
55,
118,
51,
85,
104,
100,
81,
119,
84,
80,
52,
110,
82,
78,
76,
103,
120,
52,
10,
66,
106,
121,
80,
101,
110,
90,
71,
78,
82,
120,
78,
71,
106,
72,
80,
67,
120,
50,
116,
106,
107,
71,
69,
80,
80,
83,
65,
76,
72,
109,
48,
52,
65,
69,
116,
74,
99,
97,
49,
81,
48,
117,
65,
79,
76,
86,
109,
54,
80,
108,
65,
115,
110,
101,
82,
115,
85,
97,
107,
70,
50,
97,
72,
10,
69,
110,
66,
115,
70,
118,
107,
113,
99,
43,
121,
90,
109,
111,
86,
83,
109,
107,
102,
79,
89,
111,
65,
117,
106,
73,
78,
85,
114,
69,
88,
106,
87,
73,
66,
54,
121,
53,
107,
111,
49,
119,
76,
66,
73,
88,
67,
85,
101,
70,
51,
113,
90,
102,
88,
47,
118,
80,
81,
47,
89,
49,
74,
118,
10,
54,
107,
116,
121,
102,
87,
90,
103,
67,
84,
75,
116,
55,
49,
109,
82,
99,
49,
47,
73,
50,
53,
119,
103,
51,
52,
122,
55,
104,
114,
105,
68,
81,
98,
76,
115,
114,
66,
69,
116,
43,
80,
66,
81,
85,
52,
70,
100,
122,
88,
89,
55,
72,
83,
55,
69,
88,
81,
108,
105,
117,
55,
97,
54,
10,
66,
49,
78,
108,
106,
89,
110,
103,
49,
100,
110,
74,
106,
110,
52,
116,
79,
119,
54,
104,
104,
110,
43,
122,
109,
116,
57,
53,
121,
105,
118,
89,
116,
53,
67,
97,
70,
117,
110,
49,
90,
65,
70,
72,
69,
82,
85,
68,
110,
119,
121,
53,
106,
75,
66,
48,
47,
81,
55,
80,
102,
98,
106,
65,
10,
53,
81,
76,
106,
122,
51,
69,
119,
84,
111,
108,
73,
57,
67,
110,
109,
102,
105,
48,
76,
50,
55,
105,
113,
90,
85,
107,
116,
100,
55,
80,
104,
119,
105,
84,
53,
72,
73,
55,
71,
43,
47,
86,
113,
106,
48,
55,
74,
107,
83,
85,
75,
47,
114,
51,
50,
74,
105,
43,
84,
111,
66,
51,
99,
10,
74,
105,
76,
74,
72,
103,
48,
106,
122,
103,
98,
80,
78,
65,
118,
98,
116,
68,
49,
57,
49,
100,
108,
83,
107,
116,
74,
120,
116,
86,
120,
101,
110,
55,
50,
50,
66,
69,
83,
71,
69,
81,
104,
56,
88,
97,
117,
107,
77,
102,
119,
105,
69,
97,
50,
67,
102,
86,
118,
120,
68,
97,
114,
47,
10,
66,
116,
101,
109,
110,
77,
84,
116,
122,
52,
53,
84,
75,
113,
83,
72,
49,
112,
80,
50,
101,
107,
107,
84,
78,
84,
120,
118,
52,
79,
76,
106,
54,
55,
66,
85,
65,
86,
50,
84,
120,
77,
115,
66,
98,
76,
113,
100,
103,
53,
89,
108,
113,
47,
118,
70,
105,
65,
110,
79,
112,
100,
82,
119,
10,
106,
90,
71,
88,
86,
72,
75,
54,
111,
53,
65,
57,
87,
105,
47,
88,
54,
86,
114,
89,
107,
118,
107,
47,
81,
71,
53,
71,
52,
102,
68,
71,
99,
56,
112,
84,
65,
103,
77,
66,
65,
65,
71,
106,
78,
68,
65,
121,
77,
67,
73,
71,
65,
49,
85,
100,
69,
81,
81,
98,
77,
66,
109,
67,
10,
70,
51,
74,
108,
89,
50,
57,
50,
90,
88,
74,
53,
76,
110,
78,
53,
99,
121,
53,
118,
101,
71,
108,
107,
90,
83,
53,
48,
90,
88,
78,
48,
77,
65,
119,
71,
65,
49,
85,
100,
69,
119,
69,
66,
47,
119,
81,
67,
77,
65,
65,
119,
68,
81,
89,
74,
75,
111,
90,
73,
104,
118,
99,
78,
10,
65,
81,
69,
76,
66,
81,
65,
68,
103,
103,
73,
66,
65,
71,
97,
74,
66,
105,
90,
68,
78,
89,
115,
90,
67,
67,
87,
47,
67,
102,
85,
89,
65,
53,
105,
43,
111,
84,
74,
101,
122,
111,
115,
78,
105,
70,
112,
113,
116,
48,
83,
51,
100,
86,
85,
87,
107,
100,
121,
76,
68,
112,
109,
81,
10,
73,
87,
97,
118,
100,
112,
99,
97,
80,
113,
88,
73,
102,
81,
66,
68,
103,
74,
74,
72,
69,
51,
54,
66,
78,
55,
43,
72,
53,
72,
48,
88,
108,
122,
113,
102,
111,
103,
66,
113,
121,
68,
82,
101,
109,
72,
77,
101,
84,
120,
113,
102,
76,
107,
69,
53,
71,
69,
69,
115,
72,
104,
49,
52,
10,
104,
56,
73,
74,
57,
104,
105,
117,
121,
90,
57,
80,
52,
83,
105,
72,
109,
121,
74,
104,
105,
118,
119,
115,
51,
90,
121,
107,
114,
73,
55,
100,
122,
53,
53,
114,
102,
55,
54,
84,
72,
107,
89,
79,
105,
121,
74,
80,
72,
82,
113,
55,
52,
102,
97,
50,
97,
81,
80,
117,
85,
100,
120,
56,
10,
110,
102,
67,
79,
71,
83,
88,
109,
86,
97,
77,
71,
99,
111,
103,
75,
119,
118,
104,
53,
56,
119,
99,
74,
120,
116,
54,
72,
69,
51,
88,
119,
83,
69,
117,
68,
75,
82,
111,
87,
77,
68,
106,
90,
119,
115,
79,
70,
102,
111,
107,
87,
119,
77,
66,
113,
51,
101,
49,
120,
84,
122,
90,
99,
10,
90,
102,
54,
48,
57,
70,
53,
53,
83,
105,
78,
107,
76,
83,
66,
100,
116,
112,
107,
51,
107,
105,
53,
115,
77,
70,
100,
113,
107,
99,
117,
86,
51,
78,
99,
84,
98,
85,
121,
90,
89,
43,
85,
71,
110,
111,
117,
47,
84,
72,
73,
101,
69,
53,
118,
111,
77,
107,
97,
114,
47,
114,
47,
50,
10,
68,
57,
49,
122,
112,
66,
98,
54,
109,
82,
100,
77,
82,
75,
48,
87,
102,
98,
106,
82,
120,
115,
119,
121,
86,
107,
74,
49,
98,
122,
90,
104,
110,
90,
54,
119,
88,
115,
98,
99,
55,
118,
56,
101,
56,
52,
88,
110,
56,
55,
70,
47,
106,
49,
101,
104,
82,
76,
121,
109,
87,
76,
104,
79,
10,
104,
120,
97,
50,
98,
54,
107,
55,
87,
90,
81,
89,
117,
78,
84,
49,
86,
108,
75,
51,
116,
66,
103,
109,
83,
90,
66,
53,
57,
85,
55,
86,
104,
121,
102,
54,
65,
74,
68,
43,
57,
75,
51,
56,
107,
97,
78,
49,
120,
71,
54,
65,
66,
90,
89,
121,
72,
66,
99,
101,
102,
66,
102,
51,
10,
43,
65,
90,
116,
113,
83,
83,
75,
79,
55,
52,
52,
108,
113,
88,
72,
84,
105,
109,
77,
121,
106,
122,
75,
106,
80,
78,
48,
120,
50,
65,
79,
73,
102,
112,
43,
66,
81,
50,
82,
55,
111,
72,
79,
83,
70,
54,
50,
47,
89,
101,
69,
86,
82,
103,
82,
70,
49,
103,
66,
65,
84,
51,
65,
10,
70,
102,
121,
68,
114,
110,
53,
77,
90,
55,
103,
75,
104,
117,
67,
48,
120,
67,
109,
107,
78,
57,
65,
87,
53,
118,
106,
51,
103,
108,
69,
121,
104,
71,
76,
72,
81,
53,
88,
67,
122,
79,
115,
54,
101,
102,
84,
118,
97,
86,
43,
57,
104,
121,
53,
101,
73,
87,
106,
116,
105,
102,
107,
98,
10,
122,
90,
77,
47,
80,
43,
48,
53,
100,
121,
99,
85,
78,
106,
49,
121,
119,
108,
73,
106,
117,
55,
97,
69,
48,
116,
69,
81,
75,
89,
48,
84,
49,
116,
103,
106,
121,
52,
73,
103,
43,
109,
43,
75,
110,
72,
70,
54,
108,
83,
106,
115,
70,
110,
50,
82,
75,
84,
56,
89,
57,
118,
81,
67,
10,
106,
100,
82,
121,
112,
97,
56,
79,
56,
53,
57,
98,
48,
51,
90,
52,
109,
89,
72,
82,
102,
99,
54,
102,
43,
72,
119,
112,
49,
121,
86,
105,
55,
71,
56,
69,
70,
112,
99,
108,
83,
85,
54,
79,
69,
81,
111,
101,
75,
70,
56,
48,
84,
66,
71,
111,
10,
45,
45,
45,
45,
45,
69,
78,
68,
32,
67,
69,
82,
84,
73,
70,
73,
67,
65,
84,
69,
45,
45,
45,
45,
45,
10,
],
key: [
45,
45,
45,
45,
45,
66,
69,
71,
73,
78,
32,
80,
82,
73,
86,
65,
84,
69,
32,
75,
69,
89,
45,
45,
45,
45,
45,
10,
77,
73,
73,
74,
81,
81,
73,
66,
65,
68,
65,
78,
66,
103,
107,
113,
104,
107,
105,
71,
57,
119,
48,
66,
65,
81,
69,
70,
65,
65,
83,
67,
67,
83,
115,
119,
103,
103,
107,
110,
65,
103,
69,
65,
65,
111,
73,
67,
65,
81,
68,
67,
76,
85,
85,
73,
83,
88,
103,
106,
71,
67,
49,
74,
10,
86,
47,
109,
80,
90,
106,
83,
98,
118,
56,
76,
69,
110,
97,
78,
97,
52,
49,
119,
57,
120,
101,
102,
122,
118,
105,
119,
105,
74,
108,
80,
87,
77,
105,
114,
69,
68,
73,
101,
118,
67,
104,
85,
104,
43,
115,
104,
67,
51,
77,
56,
86,
120,
53,
99,
72,
116,
117,
101,
78,
112,
52,
74,
75,
10,
84,
48,
120,
120,
87,
100,
118,
57,
117,
121,
87,
87,
72,
107,
74,
113,
74,
57,
122,
105,
118,
112,
71,
56,
122,
117,
51,
122,
107,
112,
78,
101,
113,
113,
83,
79,
122,
115,
104,
120,
119,
99,
54,
86,
57,
79,
74,
57,
43,
83,
85,
107,
113,
117,
75,
50,
103,
122,
54,
54,
48,
69,
117,
53,
10,
117,
74,
108,
113,
97,
97,
108,
108,
112,
66,
65,
57,
101,
80,
57,
83,
113,
53,
114,
102,
49,
110,
43,
49,
51,
74,
117,
102,
101,
108,
76,
106,
117,
84,
65,
78,
111,
118,
57,
79,
55,
57,
49,
73,
88,
85,
77,
69,
122,
43,
74,
48,
84,
83,
52,
77,
101,
65,
89,
56,
106,
51,
112,
50,
10,
82,
106,
85,
99,
84,
82,
111,
120,
122,
119,
115,
100,
114,
89,
53,
66,
104,
68,
122,
48,
103,
67,
120,
53,
116,
79,
65,
66,
76,
83,
88,
71,
116,
85,
78,
76,
103,
68,
105,
49,
90,
117,
106,
53,
81,
76,
74,
51,
107,
98,
70,
71,
112,
66,
100,
109,
104,
120,
74,
119,
98,
66,
98,
53,
10,
75,
110,
80,
115,
109,
90,
113,
70,
85,
112,
112,
72,
122,
109,
75,
65,
76,
111,
121,
68,
86,
75,
120,
70,
52,
49,
105,
65,
101,
115,
117,
90,
75,
78,
99,
67,
119,
83,
70,
119,
108,
72,
104,
100,
54,
109,
88,
49,
47,
55,
122,
48,
80,
50,
78,
83,
98,
43,
112,
76,
99,
110,
49,
109,
10,
89,
65,
107,
121,
114,
101,
57,
90,
107,
88,
78,
102,
121,
78,
117,
99,
73,
78,
43,
77,
43,
52,
97,
52,
103,
48,
71,
121,
55,
75,
119,
82,
76,
102,
106,
119,
85,
70,
79,
66,
88,
99,
49,
50,
79,
120,
48,
117,
120,
70,
48,
74,
89,
114,
117,
50,
117,
103,
100,
84,
90,
89,
50,
74,
10,
52,
78,
88,
90,
121,
89,
53,
43,
76,
84,
115,
79,
111,
89,
90,
47,
115,
53,
114,
102,
101,
99,
111,
114,
50,
76,
101,
81,
109,
104,
98,
112,
57,
87,
81,
66,
82,
120,
69,
86,
65,
53,
56,
77,
117,
89,
121,
103,
100,
80,
48,
79,
122,
51,
50,
52,
119,
79,
85,
67,
52,
56,
57,
120,
10,
77,
69,
54,
74,
83,
80,
81,
112,
53,
110,
52,
116,
67,
57,
117,
52,
113,
109,
86,
74,
76,
88,
101,
122,
52,
99,
73,
107,
43,
82,
121,
79,
120,
118,
118,
49,
97,
111,
57,
79,
121,
90,
69,
108,
67,
118,
54,
57,
57,
105,
89,
118,
107,
54,
65,
100,
51,
67,
89,
105,
121,
82,
52,
78,
10,
73,
56,
52,
71,
122,
122,
81,
76,
50,
55,
81,
57,
102,
100,
88,
90,
85,
112,
76,
83,
99,
98,
86,
99,
88,
112,
43,
57,
116,
103,
82,
69,
104,
104,
69,
73,
102,
70,
50,
114,
112,
68,
72,
56,
73,
104,
71,
116,
103,
110,
49,
98,
56,
81,
50,
113,
47,
119,
98,
88,
112,
112,
122,
69,
10,
55,
99,
43,
79,
85,
121,
113,
107,
104,
57,
97,
84,
57,
110,
112,
74,
69,
122,
85,
56,
98,
43,
68,
105,
52,
43,
117,
119,
86,
65,
70,
100,
107,
56,
84,
76,
65,
87,
121,
54,
110,
89,
79,
87,
74,
97,
118,
55,
120,
89,
103,
74,
122,
113,
88,
85,
99,
73,
50,
82,
108,
49,
82,
121,
10,
117,
113,
79,
81,
80,
86,
111,
118,
49,
43,
108,
97,
50,
74,
76,
53,
80,
48,
66,
117,
82,
117,
72,
119,
120,
110,
80,
75,
85,
119,
73,
68,
65,
81,
65,
66,
65,
111,
73,
67,
65,
72,
88,
122,
106,
100,
87,
69,
50,
52,
100,
108,
55,
107,
120,
53,
78,
107,
49,
88,
90,
109,
109,
100,
10,
121,
76,
65,
113,
109,
99,
112,
105,
66,
101,
119,
52,
82,
98,
103,
89,
77,
122,
76,
106,
87,
86,
88,
112,
122,
51,
83,
98,
114,
43,
103,
121,
70,
98,
103,
52,
89,
71,
72,
78,
81,
55,
89,
86,
77,
98,
99,
114,
56,
101,
104,
111,
99,
111,
113,
49,
78,
78,
103,
43,
72,
53,
89,
51,
10,
107,
112,
78,
110,
43,
105,
119,
70,
51,
118,
87,
86,
107,
87,
51,
66,
118,
87,
87,
83,
67,
104,
99,
100,
84,
83,
121,
54,
105,
54,
81,
101,
101,
71,
69,
55,
67,
107,
73,
47,
106,
56,
75,
65,
85,
52,
116,
79,
109,
108,
74,
99,
113,
85,
77,
57,
48,
87,
79,
69,
110,
55,
80,
119,
10,
80,
122,
85,
52,
82,
78,
66,
120,
69,
88,
107,
98,
69,
83,
84,
108,
120,
49,
114,
79,
115,
88,
57,
72,
82,
78,
116,
106,
119,
102,
97,
118,
87,
104,
110,
86,
72,
82,
114,
86,
78,
78,
103,
48,
56,
100,
51,
57,
73,
114,
121,
121,
52,
48,
55,
112,
81,
117,
88,
89,
75,
67,
116,
78,
10,
101,
53,
69,
104,
108,
105,
74,
83,
77,
55,
122,
75,
66,
122,
111,
47,
70,
122,
116,
102,
115,
50,
122,
76,
79,
118,
43,
110,
43,
99,
113,
110,
111,
113,
109,
87,
70,
86,
76,
119,
109,
50,
69,
118,
50,
69,
56,
100,
105,
97,
80,
107,
53,
79,
101,
105,
86,
66,
122,
76,
67,
99,
81,
109,
10,
56,
122,
84,
102,
78,
53,
74,
112,
116,
69,
48,
113,
113,
67,
77,
107,
65,
79,
83,
54,
115,
117,
90,
115,
109,
110,
56,
47,
49,
104,
74,
79,
47,
102,
98,
43,
112,
99,
85,
83,
47,
106,
76,
89,
78,
110,
115,
106,
74,
73,
50,
69,
70,
73,
121,
86,
108,
99,
68,
70,
83,
51,
104,
114,
10,
97,
107,
89,
115,
116,
112,
80,
75,
110,
77,
77,
122,
55,
77,
72,
98,
71,
76,
112,
67,
82,
85,
121,
43,
81,
107,
49,
79,
117,
120,
88,
119,
65,
79,
74,
116,
104,
73,
113,
48,
84,
85,
90,
105,
90,
90,
57,
71,
74,
84,
100,
120,
99,
49,
102,
51,
115,
88,
80,
119,
103,
119,
56,
54,
10,
117,
71,
98,
119,
55,
67,
53,
84,
120,
84,
98,
43,
48,
103,
113,
74,
43,
49,
43,
73,
70,
77,
111,
75,
113,
51,
77,
88,
87,
69,
110,
72,
69,
98,
99,
86,
119,
90,
119,
53,
78,
106,
76,
57,
110,
65,
115,
83,
111,
81,
69,
43,
80,
122,
84,
87,
71,
104,
110,
100,
86,
70,
67,
119,
10,
65,
67,
98,
43,
89,
113,
71,
116,
79,
69,
104,
75,
68,
80,
70,
116,
49,
117,
80,
103,
88,
81,
89,
98,
68,
115,
104,
86,
68,
70,
101,
82,
118,
119,
114,
84,
85,
115,
113,
83,
83,
65,
109,
119,
120,
86,
65,
43,
98,
89,
98,
55,
120,
73,
114,
100,
105,
85,
107,
98,
71,
65,
43,
115,
10,
118,
69,
90,
43,
66,
99,
108,
76,
56,
82,
71,
53,
102,
67,
113,
112,
119,
111,
53,
57,
101,
113,
47,
117,
47,
81,
105,
115,
116,
77,
113,
111,
81,
114,
72,
82,
85,
65,
72,
98,
83,
100,
55,
49,
81,
119,
116,
111,
48,
81,
86,
68,
85,
116,
47,
65,
66,
75,
71,
77,
98,
69,
78,
57,
10,
51,
104,
75,
50,
105,
102,
73,
84,
102,
97,
75,
102,
111,
111,
107,
72,
82,
73,
116,
86,
72,
50,
79,
90,
103,
81,
83,
43,
122,
49,
68,
106,
53,
89,
67,
115,
116,
76,
107,
78,
111,
78,
79,
99,
69,
99,
43,
103,
80,
55,
55,
67,
108,
43,
106,
67,
87,
54,
98,
82,
113,
48,
78,
114,
10,
110,
118,
80,
114,
118,
110,
118,
43,
106,
104,
84,
72,
108,
103,
83,
50,
106,
57,
84,
104,
65,
111,
73,
66,
65,
81,
68,
119,
115,
100,
66,
103,
75,
105,
51,
115,
107,
49,
117,
51,
68,
121,
56,
98,
74,
108,
122,
101,
52,
114,
89,
119,
109,
88,
100,
118,
75,
65,
105,
68,
70,
83,
54,
70,
10,
118,
53,
114,
56,
66,
84,
84,
54,
104,
54,
66,
112,
53,
106,
87,
65,
105,
108,
87,
57,
72,
48,
85,
72,
89,
71,
86,
72,
113,
73,
71,
53,
51,
54,
97,
111,
113,
49,
81,
49,
75,
84,
86,
72,
105,
89,
84,
82,
102,
81,
121,
55,
73,
100,
118,
120,
89,
75,
48,
54,
106,
108,
79,
117,
10,
49,
102,
99,
103,
99,
50,
52,
117,
69,
122,
56,
106,
116,
115,
55,
76,
115,
90,
68,
47,
47,
116,
73,
43,
69,
73,
69,
118,
103,
116,
107,
98,
80,
80,
119,
90,
82,
121,
121,
100,
119,
51,
108,
79,
113,
98,
85,
74,
84,
71,
49,
84,
48,
78,
74,
89,
97,
78,
84,
111,
108,
86,
68,
50,
10,
77,
115,
84,
50,
117,
85,
87,
100,
117,
53,
120,
49,
115,
113,
103,
111,
87,
120,
100,
98,
43,
71,
104,
87,
57,
116,
51,
112,
100,
100,
100,
83,
103,
88,
56,
99,
74,
120,
77,
102,
120,
103,
80,
83,
65,
99,
105,
100,
80,
86,
110,
119,
101,
98,
88,
118,
121,
113,
81,
86,
74,
110,
85,
66,
10,
112,
118,
67,
101,
97,
104,
104,
114,
52,
89,
119,
121,
120,
109,
90,
107,
104,
114,
78,
115,
70,
75,
71,
78,
114,
110,
90,
111,
73,
84,
105,
85,
122,
78,
48,
50,
43,
56,
82,
82,
97,
119,
103,
87,
101,
53,
89,
102,
113,
104,
117,
106,
72,
89,
79,
100,
75,
87,
102,
48,
52,
83,
117,
71,
10,
111,
118,
84,
109,
74,
107,
66,
89,
112,
99,
53,
109,
89,
100,
77,
119,
88,
106,
104,
120,
84,
108,
83,
69,
79,
70,
48,
89,
113,
113,
109,
57,
69,
56,
57,
105,
69,
98,
79,
82,
121,
87,
114,
99,
108,
89,
65,
76,
65,
111,
73,
66,
65,
81,
68,
79,
104,
106,
86,
54,
86,
87,
116,
68,
10,
113,
98,
55,
71,
67,
117,
48,
101,
111,
75,
118,
85,
116,
106,
122,
66,
107,
122,
80,
73,
102,
75,
74,
87,
86,
100,
110,
50,
85,
104,
100,
69,
50,
102,
100,
51,
55,
114,
113,
55,
68,
97,
88,
82,
115,
47,
85,
87,
101,
70,
75,
113,
90,
76,
117,
97,
108,
80,
72,
98,
72,
105,
111,
56,
10,
76,
119,
119,
83,
87,
82,
122,
68,
114,
53,
108,
83,
121,
112,
103,
84,
116,
69,
65,
47,
100,
77,
122,
71,
87,
116,
112,
111,
75,
54,
118,
53,
98,
85,
115,
78,
49,
49,
79,
120,
57,
113,
104,
67,
113,
73,
89,
119,
57,
104,
89,
87,
56,
54,
118,
71,
68,
47,
110,
49,
49,
77,
113,
87,
10,
72,
118,
65,
86,
53,
115,
49,
78,
80,
113,
122,
84,
99,
116,
73,
73,
79,
81,
115,
122,
113,
88,
90,
97,
116,
99,
72,
117,
107,
67,
54,
102,
73,
87,
116,
106,
113,
120,
122,
100,
52,
88,
72,
122,
104,
112,
89,
72,
102,
118,
71,
113,
90,
120,
65,
110,
112,
76,
65,
49,
97,
83,
112,
47,
10,
54,
50,
56,
48,
97,
50,
67,
87,
103,
121,
67,
111,
78,
89,
48,
99,
114,
89,
75,
80,
114,
87,
76,
81,
100,
102,
67,
106,
107,
49,
72,
110,
98,
119,
110,
49,
112,
72,
55,
115,
75,
53,
70,
47,
113,
103,
113,
76,
68,
86,
68,
90,
103,
116,
69,
112,
118,
105,
48,
75,
65,
120,
74,
57,
10,
47,
106,
52,
47,
114,
53,
119,
78,
56,
54,
52,
112,
71,
56,
85,
52,
90,
115,
100,
109,
77,
82,
90,
53,
85,
79,
76,
111,
51,
43,
108,
88,
88,
83,
89,
108,
73,
104,
55,
77,
66,
71,
118,
66,
77,
79,
68,
102,
50,
48,
79,
86,
55,
107,
103,
109,
100,
122,
71,
76,
56,
69,
113,
87,
10,
117,
112,
43,
112,
112,
80,
66,
70,
89,
109,
80,
90,
65,
111,
73,
66,
65,
72,
72,
56,
56,
57,
56,
109,
43,
112,
114,
73,
97,
113,
51,
50,
72,
101,
118,
119,
114,
104,
106,
77,
75,
49,
89,
50,
120,
56,
52,
51,
109,
108,
104,
104,
53,
113,
100,
109,
120,
52,
74,
70,
65,
81,
77,
73,
10,
121,
104,
112,
56,
56,
75,
101,
81,
87,
43,
120,
49,
54,
53,
52,
68,
115,
57,
78,
77,
53,
83,
114,
101,
49,
84,
71,
90,
43,
103,
89,
51,
72,
115,
108,
106,
81,
105,
104,
110,
51,
87,
105,
101,
76,
50,
104,
80,
122,
90,
111,
109,
110,
49,
68,
48,
101,
100,
87,
47,
100,
78,
87,
84,
10,
80,
100,
113,
118,
68,
53,
53,
57,
50,
70,
113,
48,
70,
109,
78,
89,
77,
54,
89,
74,
104,
77,
73,
87,
106,
112,
108,
50,
69,
74,
89,
105,
90,
99,
101,
119,
76,
86,
70,
98,
66,
89,
101,
108,
74,
73,
106,
69,
70,
78,
105,
86,
74,
107,
79,
67,
90,
97,
56,
109,
114,
79,
90,
51,
10,
88,
68,
67,
78,
84,
72,
47,
89,
75,
43,
98,
100,
98,
75,
83,
75,
119,
121,
116,
50,
75,
71,
81,
108,
101,
99,
111,
101,
89,
122,
115,
118,
55,
115,
48,
57,
70,
81,
73,
75,
100,
82,
97,
105,
118,
86,
118,
122,
106,
117,
79,
74,
85,
43,
53,
86,
88,
105,
122,
88,
116,
47,
115,
88,
10,
86,
79,
115,
83,
57,
74,
106,
49,
66,
102,
72,
81,
106,
51,
88,
109,
119,
113,
75,
114,
52,
80,
122,
118,
84,
72,
66,
109,
113,
87,
71,
90,
109,
55,
110,
121,
67,
51,
87,
120,
102,
79,
99,
120,
54,
103,
117,
54,
43,
85,
87,
75,
103,
97,
43,
65,
97,
120,
83,
51,
49,
115,
57,
107,
10,
114,
113,
118,
106,
57,
106,
48,
52,
48,
101,
103,
118,
72,
87,
50,
69,
116,
43,
111,
102,
111,
77,
85,
118,
112,
89,
52,
102,
76,
102,
105,
68,
57,
47,
122,
66,
80,
71,
77,
67,
103,
103,
69,
65,
77,
104,
117,
48,
67,
47,
116,
47,
114,
65,
111,
97,
90,
72,
105,
119,
77,
90,
99,
119,
10,
79,
99,
115,
66,
107,
77,
73,
68,
100,
87,
104,
119,
66,
116,
121,
76,
112,
115,
48,
105,
73,
105,
49,
48,
99,
112,
47,
48,
112,
72,
47,
97,
78,
76,
103,
105,
66,
104,
75,
112,
107,
47,
103,
101,
52,
74,
71,
48,
52,
77,
73,
47,
52,
120,
120,
84,
74,
72,
121,
69,
107,
121,
73,
101,
10,
53,
69,
121,
78,
101,
51,
74,
110,
105,
68,
51,
120,
108,
85,
114,
106,
53,
82,
82,
55,
104,
71,
103,
49,
74,
113,
87,
82,
66,
65,
80,
104,
114,
86,
75,
119,
82,
70,
55,
103,
50,
121,
55,
67,
66,
97,
116,
73,
114,
90,
71,
48,
109,
90,
110,
109,
68,
74,
112,
85,
81,
54,
75,
113,
10,
85,
104,
57,
119,
119,
70,
102,
43,
49,
77,
107,
81,
69,
49,
65,
52,
80,
111,
97,
122,
71,
82,
56,
70,
114,
69,
103,
80,
70,
47,
75,
107,
77,
68,
55,
43,
86,
100,
70,
67,
70,
50,
77,
115,
77,
48,
73,
116,
69,
114,
104,
121,
102,
70,
107,
71,
81,
85,
83,
119,
83,
98,
83,
70,
10,
68,
117,
114,
51,
85,
118,
98,
121,
84,
54,
80,
48,
55,
114,
107,
43,
54,
113,
70,
75,
53,
81,
107,
50,
57,
71,
101,
49,
57,
57,
49,
87,
84,
120,
66,
86,
119,
73,
107,
53,
106,
90,
122,
69,
51,
81,
106,
115,
97,
112,
112,
50,
79,
83,
87,
102,
117,
116,
111,
68,
49,
103,
114,
67,
10,
118,
101,
121,
106,
47,
70,
120,
76,
107,
119,
114,
106,
57,
103,
108,
102,
70,
76,
117,
72,
74,
82,
66,
80,
117,
103,
67,
43,
85,
105,
102,
99,
99,
83,
74,
72,
51,
87,
105,
89,
122,
70,
55,
108,
71,
74,
103,
103,
54,
71,
65,
106,
54,
78,
66,
99,
47,
115,
56,
114,
69,
78,
113,
102,
10,
101,
81,
75,
67,
65,
81,
65,
100,
90,
47,
56,
57,
119,
110,
118,
69,
106,
66,
73,
100,
114,
84,
80,
66,
101,
53,
89,
89,
56,
69,
110,
66,
71,
116,
108,
111,
122,
84,
111,
114,
108,
43,
74,
99,
81,
119,
105,
53,
115,
105,
82,
110,
97,
78,
57,
83,
74,
49,
76,
90,
103,
43,
82,
52,
10,
70,
51,
111,
87,
114,
107,
72,
121,
86,
119,
70,
99,
122,
72,
69,
120,
85,
88,
90,
118,
70,
69,
88,
97,
53,
68,
75,
71,
87,
48,
112,
66,
55,
117,
55,
105,
110,
99,
47,
104,
112,
120,
48,
86,
51,
120,
97,
98,
65,
71,
115,
83,
75,
116,
80,
113,
43,
121,
54,
116,
71,
114,
68,
53,
10,
111,
43,
57,
66,
87,
65,
57,
104,
115,
119,
113,
74,
66,
82,
75,
76,
118,
67,
119,
80,
106,
75,
74,
69,
84,
104,
48,
120,
90,
85,
52,
102,
98,
88,
119,
122,
69,
120,
84,
67,
78,
111,
79,
88,
74,
47,
105,
55,
100,
108,
110,
54,
55,
110,
86,
105,
87,
69,
68,
98,
73,
52,
113,
109,
10,
50,
49,
89,
122,
65,
83,
120,
68,
98,
109,
79,
55,
89,
106,
122,
77,
50,
84,
118,
57,
81,
78,
97,
82,
89,
88,
48,
77,
88,
119,
101,
51,
108,
82,
89,
73,
49,
118,
111,
55,
102,
68,
107,
75,
54,
82,
89,
82,
49,
75,
72,
104,
121,
69,
68,
82,
119,
109,
78,
85,
120,
70,
57,
84,
10,
109,
65,
51,
85,
116,
53,
98,
78,
87,
112,
118,
103,
101,
47,
98,
112,
77,
102,
80,
75,
99,
81,
102,
115,
113,
82,
121,
79,
69,
81,
70,
120,
111,
72,
68,
107,
79,
119,
100,
103,
121,
119,
49,
121,
53,
103,
106,
101,
72,
86,
114,
83,
117,
74,
47,
66,
101,
43,
56,
54,
112,
75,
74,
53,
10,
101,
49,
55,
77,
121,
47,
120,
87,
107,
121,
114,
109,
83,
83,
78,
108,
50,
84,
113,
83,
110,
84,
83,
116,
65,
76,
73,
67,
10,
45,
45,
45,
45,
45,
69,
78,
68,
32,
80,
82,
73,
86,
65,
84,
69,
32,
75,
69,
89,
45,
45,
45,
45,
45,
10,
],
},
],
recovery_silo: RecoverySiloConfig {
silo_name: Name(
"recovery",
),
user_name: UserId(
"recovery",
),
user_password_hash: NewPasswordHash(
PasswordHashString {
string: "$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY",
encoding: B64,
},
),
},
}
|
| 43 | 2023-05-24T20:18:15.198Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b",
size: 10200547328,
allocated: 231936,
free: 10200315392,
health: Online,
}
|
| 44 | 2023-05-24T20:18:15.198Z | INFO | SledAgent (StorageManager): Upserting synthetic disk for: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }
|
| 45 | 2023-05-24T20:18:15.198Z | INFO | SledAgent (RSS): RSS configuration has not been fully applied yet
|
| 46 | 2023-05-24T20:18:15.261Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
size: 10200547328,
allocated: 236544,
free: 10200310784,
health: Online,
}
|
| 47 | 2023-05-24T20:18:15.261Z | INFO | SledAgent (StorageManager): Upserting synthetic disk for: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }
|
| 48 | 2023-05-24T20:18:15.261Z | INFO | SledAgent (RSS): Creating new allocation plan
|
| 49 | 2023-05-24T20:18:15.261Z | INFO | SledAgent (RSS): Serializing plan
|
| 50 | 2023-05-24T20:18:15.261Z | INFO | SledAgent (RSS): Creating plan for the sled at fdb0:18c0:4d0d:a02a::1
|
| 51 | 2023-05-24T20:18:15.323Z | INFO | SledAgent (StorageManager): Storage manager processing zpool: ZpoolInfo {
name: "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
size: 10200547328,
allocated: 235008,
free: 10200312320,
health: Online,
}
|
| 52 | 2023-05-24T20:18:15.323Z | INFO | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
|
| 53 | 2023-05-24T20:18:15.323Z | INFO | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
|
| 54 | 2023-05-24T20:18:15.323Z | INFO | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
|
| 55 | 2023-05-24T20:18:15.323Z | INFO | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
|
| 56 | 2023-05-24T20:18:15.323Z | INFO | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
|
| 57 | 2023-05-24T20:18:15.323Z | INFO | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
|
| 58 | 2023-05-24T20:18:15.326Z | INFO | SledAgent (RSS): Sled plan written to storage
|
| 59 | 2023-05-24T20:18:15.326Z | INFO | SledAgent (RSS): Skipping rack secret creation (only one sled present)
|
| 60 | 2023-05-24T20:18:15.326Z | INFO | SledAgent (BootstrapAgentRssHandler): Received initialization request from RSS
request = SledAgentRequest { id: 6f0f865b-7d3c-40ef-a8b9-18c66c253d85, rack_id: 327b9e22-d506-4e4b-8546-82149102718d, ntp_servers: ["ntp.eng.oxide.computer"], dns_servers: ["1.1.1.1", "9.9.9.9"], subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 61 | 2023-05-24T20:18:15.326Z | INFO | SledAgent (BootstrapAgentRssHandler): No SP available; proceeding without sprockets auth
BootstrapAgentClient = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 62 | 2023-05-24T20:18:15.326Z | INFO | SledAgent (sprockets (BootstrapAgent)): Accepted connection
remote_addr = [fdb0:18c0:4d0d:a02a::1]:41388
|
| 63 | 2023-05-24T20:18:15.326Z | INFO | SledAgent (sprockets (BootstrapAgent)): No SP available; proceeding without sprockets auth
remote_addr = [fdb0:18c0:4d0d:a02a::1]:41388
|
| 64 | 2023-05-24T20:18:15.326Z | INFO | SledAgent (BootstrapAgent): Loading Sled Agent: SledAgentRequest { id: 6f0f865b-7d3c-40ef-a8b9-18c66c253d85, rack_id: 327b9e22-d506-4e4b-8546-82149102718d, ntp_servers: ["ntp.eng.oxide.computer"], dns_servers: ["1.1.1.1", "9.9.9.9"], subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }
|
| 65 | 2023-05-24T20:18:15.326Z | INFO | SledAgent: setting up sled agent server
|
| 66 | 2023-05-24T20:18:15.326Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53]
|
| 67 | 2023-05-24T20:18:15.327Z | INFO | SledAgent: created sled agent
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 68 | 2023-05-24T20:18:15.851Z | INFO | SledAgent: using '[AddrObject { interface: "net0", name: "ll" }, AddrObject { interface: "net1", name: "ll" }]' as data links for xde driver
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 69 | 2023-05-24T20:18:15.851Z | INFO | SledAgent (HardwareManager): Creating HardwareManager
|
| 70 | 2023-05-24T20:18:15.854Z | INFO | SledAgent (HardwareManager): Generated i86pc baseboard Pc { identifier: "da005c867f5e4d1daa7f756de235d90e", model: "i86pc" }
|
| 71 | 2023-05-24T20:18:15.854Z | WARN | SledAgent (HardwareManager): Device is not a Gimlet (i86pc), proceeding with null hardware view
|
| 72 | 2023-05-24T20:18:15.854Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53]
|
| 73 | 2023-05-24T20:18:15.854Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 74 | 2023-05-24T20:18:15.854Z | INFO | SledAgent (HardwareManager): Monitoring for hardware updates
|
| 75 | 2023-05-24T20:18:15.854Z | INFO | SledAgent: Performing full hardware scan
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 76 | 2023-05-24T20:18:15.854Z | INFO | SledAgent (dropshot (SledAgent)): listening
local_addr = [fd00:1122:3344:101::1]:12345
|
| 77 | 2023-05-24T20:18:15.854Z | INFO | SledAgent (ServiceManager): Ensuring scrimlet services (enabling services)
|
| 78 | 2023-05-24T20:18:15.854Z | INFO | SledAgent (BootstrapAgent): Sled Agent loaded; recording configuration
|
| 79 | 2023-05-24T20:18:15.855Z | INFO | SledAgent: Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
prefix = Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }
|
| 80 | 2023-05-24T20:18:15.855Z | INFO | SledAgent (sprockets (BootstrapAgent)): Connection closed
remote_addr = [fdb0:18c0:4d0d:a02a::1]:41388
|
| 81 | 2023-05-24T20:18:15.855Z | INFO | SledAgent: failed to notify nexus about sled agent
error = proto error: io error: No route to host (os error 148)
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 82 | 2023-05-24T20:18:15.855Z | INFO | SledAgent (BootstrapAgentRssHandler): Peer agent initialized
peer = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 83 | 2023-05-24T20:18:15.855Z | INFO | SledAgent (BootstrapAgentRssHandler): Initialized sled agent
target_sled = [fdb0:18c0:4d0d:a02a::1]:12346
|
| 84 | 2023-05-24T20:18:15.855Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:62420
|
| 85 | 2023-05-24T20:18:15.856Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:62420
req_id = e76b5f7b-e063-4008-b99f-52212f456801
response_code = 200
uri = /zpools
|
| 86 | 2023-05-24T20:18:15.856Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:35489
|
| 87 | 2023-05-24T20:18:15.856Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:35489
req_id = bda27c86-3b47-4329-b8d6-9f5b9303265c
response_code = 200
uri = /sled-role
|
| 88 | 2023-05-24T20:18:15.857Z | INFO | SledAgent (RSS): Service plan written to storage
|
| 89 | 2023-05-24T20:18:15.858Z | INFO | SledAgent (RSS): sending service requests...
|
| 90 | 2023-05-24T20:18:15.858Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }]
|
| 91 | 2023-05-24T20:18:15.858Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:45525
|
| 92 | 2023-05-24T20:18:15.858Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: InternalDns
|
| 93 | 2023-05-24T20:18:15.858Z | INFO | SledAgent (ServiceManager): Service zone internal_dns does not yet exist
|
| 94 | 2023-05-24T20:18:15.873Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_internal_dns
|
| 95 | 2023-05-24T20:18:15.890Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_internal_dns
|
| 96 | 2023-05-24T20:18:15.915Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 97 | 2023-05-24T20:18:16.031Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 98 | 2023-05-24T20:18:16.133Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 99 | 2023-05-24T20:18:16.278Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 100 | 2023-05-24T20:18:16.553Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 101 | 2023-05-24T20:18:16.834Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 102 | 2023-05-24T20:18:17.049Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 103 | 2023-05-24T20:18:17.443Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 104 | 2023-05-24T20:18:17.516Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_internal_dns
|
| 105 | 2023-05-24T20:18:17.785Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_switch
|
| 106 | 2023-05-24T20:18:18.019Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 107 | 2023-05-24T20:18:18.528Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 108 | 2023-05-24T20:18:19.061Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 109 | 2023-05-24T20:18:19.566Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 110 | 2023-05-24T20:18:20.452Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 111 | 2023-05-24T20:18:21.829Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 112 | 2023-05-24T20:18:22.091Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:1::1 exists
|
| 113 | 2023-05-24T20:18:22.091Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))
zone = oxz_internal_dns
|
| 114 | 2023-05-24T20:18:22.630Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:1::1 exists - OK
|
| 115 | 2023-05-24T20:18:22.630Z | INFO | SledAgent (ServiceManager): GZ addresses: [
fd00:1122:3344:1::2,
]
|
| 116 | 2023-05-24T20:18:22.630Z | INFO | SledAgent (ServiceManager): Ensuring GZ address fd00:1122:3344:1::2 exists
|
| 117 | 2023-05-24T20:18:23.248Z | INFO | SledAgent (ServiceManager): Setting up internal-dns service
|
| 118 | 2023-05-24T20:18:23.349Z | INFO | SledAgent (ServiceManager): Sending prefix to ddmd for advertisement
DdmAdminClient = [::1]:8000
prefix = Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }
|
| 119 | 2023-05-24T20:18:23.350Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 120 | 2023-05-24T20:18:23.354Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:45525
req_id = 0e930f8a-05eb-4d27-9e91-0c569122e936
response_code = 204
uri = /services
|
| 121 | 2023-05-24T20:18:23.354Z | INFO | SledAgent (RSS): Configuring DNS server
dns_config_addr = [fd00:1122:3344:1::1]:5353
|
| 122 | 2023-05-24T20:18:23.354Z | WARN | SledAgent (RSS): failed to write DNS configuration (will retry in 87.004637ms)
dns_config_addr = [fd00:1122:3344:1::1]:5353
error_message = Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)
|
| 123 | 2023-05-24T20:18:23.445Z | INFO | SledAgent (RSS): Configured all DNS servers
|
| 124 | 2023-05-24T20:18:23.445Z | INFO | SledAgent (RSS): sending service requests...
|
| 125 | 2023-05-24T20:18:23.445Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 126 | 2023-05-24T20:18:23.445Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:57093
|
| 127 | 2023-05-24T20:18:23.445Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Ntp
|
| 128 | 2023-05-24T20:18:23.445Z | INFO | SledAgent (ServiceManager): Service zone ntp does not yet exist
|
| 129 | 2023-05-24T20:18:23.454Z | INFO | SledAgent (PortManager): Created OPTE port
port = Port { inner: PortInner { name: "opte0", _ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 213, 81]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: "vopte0" } }
|
| 130 | 2023-05-24T20:18:23.454Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
service = ServiceZoneService { id: 0d607e9a-f782-4173-b84a-04929f7be674, details: BoundaryNtp { ntp_servers: ["ntp.eng.oxide.computer"], dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, kind: Service { id: 0d607e9a-f782-4173-b84a-04929f7be674 }, name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 192.168.1.22, first_port: 0, last_port: 16383 } } }
|
| 131 | 2023-05-24T20:18:30.322Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 132 | 2023-05-24T20:18:30.652Z | INFO | SledAgent (ServiceManager): Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone
|
| 133 | 2023-05-24T20:18:30.653Z | INFO | SledAgent (ServiceManager): Adding bootstrap address
zone = oxz_switch
|
| 134 | 2023-05-24T20:18:31.194Z | INFO | SledAgent (ServiceManager): Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe88:e715
|
| 135 | 2023-05-24T20:18:31.215Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
| 136 | 2023-05-24T20:18:31.235Z | WARN | SledAgent (ServiceManager): Failed to initialize switch: Failed to do 'Adding Route' by running command in zone: Error running command in zone 'oxz_switch': Command [/usr/sbin/zlogin oxz_switch /usr/sbin/route add -inet6 default -inet6 fd00:1122:3344:101::1] executed and failed with status: exit status: 128 stdout: add net default: gateway fd00:1122:3344:101::1: Network is unreachable
stderr:
|
| 137 | 2023-05-24T20:18:31.235Z | INFO | SledAgent (ServiceManager): Enabling switch zone (already underway)
|
| 138 | 2023-05-24T20:18:32.258Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_switch in state ShuttingDown
|
| 139 | 2023-05-24T20:18:32.258Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_switch
|
| 140 | 2023-05-24T20:18:32.275Z | WARN | SledAgent (ServiceManager): Failed to initialize switch: Failed to install zone 'oxz_switch' from '/opt/oxide/switch.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_switch': Failed to parse command output: exit code 1
stdout:
stderr:
zoneadm: zone 'oxz_switch': uninstall operation is invalid for shutting_down zones.
|
| 141 | 2023-05-24T20:18:32.635Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_switch
|
| 142 | 2023-05-24T20:18:32.635Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_switch
|
| 143 | 2023-05-24T20:18:33.298Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_switch
|
| 144 | 2023-05-24T20:18:33.315Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_switch
|
| 145 | 2023-05-24T20:18:35.665Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_switch
|
| 146 | 2023-05-24T20:18:38.456Z | WARN | SledAgent (ServiceManager): failed to create NAT entry for service
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12224/nat/ipv4/192.168.1.22/0/16383): operation timed out
service = ServiceZoneService { id: 0d607e9a-f782-4173-b84a-04929f7be674, details: BoundaryNtp { ntp_servers: ["ntp.eng.oxide.computer"], dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, kind: Service { id: 0d607e9a-f782-4173-b84a-04929f7be674 }, name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 192.168.1.22, first_port: 0, last_port: 16383 } } }
|
| 147 | 2023-05-24T20:18:38.552Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
service = ServiceZoneService { id: 0d607e9a-f782-4173-b84a-04929f7be674, details: BoundaryNtp { ntp_servers: ["ntp.eng.oxide.computer"], dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, kind: Service { id: 0d607e9a-f782-4173-b84a-04929f7be674 }, name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 192.168.1.22, first_port: 0, last_port: 16383 } } }
|
| 148 | 2023-05-24T20:18:48.489Z | INFO | SledAgent (ServiceManager): Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone
|
| 149 | 2023-05-24T20:18:48.489Z | INFO | SledAgent (ServiceManager): Adding bootstrap address
zone = oxz_switch
|
| 150 | 2023-05-24T20:18:49.030Z | INFO | SledAgent (ServiceManager): Forwarding bootstrap traffic via oxBootstrap2 to fe80::8:20ff:fe88:e715
|
| 151 | 2023-05-24T20:18:49.052Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists
|
| 152 | 2023-05-24T20:18:49.052Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))
zone = oxz_switch
|
| 153 | 2023-05-24T20:18:49.591Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::2 exists - OK
|
| 154 | 2023-05-24T20:18:49.591Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
| 155 | 2023-05-24T20:18:49.745Z | INFO | SledAgent (ServiceManager): Setting up dendrite service
|
| 156 | 2023-05-24T20:18:50.160Z | INFO | SledAgent (ServiceManager): Setting up MGS service
|
| 157 | 2023-05-24T20:18:50.414Z | INFO | SledAgent (ServiceManager): Setting up wicketd service
|
| 158 | 2023-05-24T20:18:50.500Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 34.645970794s
|
| 159 | 2023-05-24T20:18:50.501Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 35.178126393s
|
| 160 | 2023-05-24T20:18:50.501Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 35.178332409s
|
| 161 | 2023-05-24T20:18:50.501Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 35.362171126s
|
| 162 | 2023-05-24T20:18:50.501Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 35.178657628s
|
| 163 | 2023-05-24T20:18:50.502Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 35.178746541s
|
| 164 | 2023-05-24T20:18:50.502Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 35.362504655s
|
| 165 | 2023-05-24T20:18:50.502Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 35.362827154s
|
| 166 | 2023-05-24T20:18:50.502Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 35.179203122s
|
| 167 | 2023-05-24T20:18:50.502Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 35.362865465s
|
| 168 | 2023-05-24T20:18:50.502Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 35.179313705s
|
| 169 | 2023-05-24T20:18:50.703Z | INFO | SledAgent (ServiceManager): Setting up Maghemite service
|
| 170 | 2023-05-24T20:18:53.083Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 171 | 2023-05-24T20:18:53.554Z | WARN | SledAgent (ServiceManager): failed to create NAT entry for service
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12224/nat/ipv4/192.168.1.22/0/16383): operation timed out
service = ServiceZoneService { id: 0d607e9a-f782-4173-b84a-04929f7be674, details: BoundaryNtp { ntp_servers: ["ntp.eng.oxide.computer"], dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, kind: Service { id: 0d607e9a-f782-4173-b84a-04929f7be674 }, name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 192.168.1.22, first_port: 0, last_port: 16383 } } }
|
| 172 | 2023-05-24T20:18:53.724Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
service = ServiceZoneService { id: 0d607e9a-f782-4173-b84a-04929f7be674, details: BoundaryNtp { ntp_servers: ["ntp.eng.oxide.computer"], dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, kind: Service { id: 0d607e9a-f782-4173-b84a-04929f7be674 }, name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 192.168.1.22, first_port: 0, last_port: 16383 } } }
|
| 173 | 2023-05-24T20:18:53.741Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_ntp
|
| 174 | 2023-05-24T20:18:53.758Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_ntp
|
| 175 | 2023-05-24T20:18:54.553Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_ntp
|
| 176 | 2023-05-24T20:18:59.338Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::c exists
|
| 177 | 2023-05-24T20:18:59.338Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::c, prefix: 64 }))
zone = oxz_ntp
|
| 178 | 2023-05-24T20:18:59.879Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::c exists - OK
|
| 179 | 2023-05-24T20:18:59.879Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
| 180 | 2023-05-24T20:19:00.036Z | INFO | SledAgent (ServiceManager): Set up NTP service boundary=true, Servers=["ntp.eng.oxide.computer"]
|
| 181 | 2023-05-24T20:19:00.090Z | INFO | SledAgent (ServiceManager): Ensuring address for OPTE port
zone = oxz_ntp
|
| 182 | 2023-05-24T20:19:05.391Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:57093
req_id = d2f7a1dc-3121-4848-8a61-d2d7169e66e2
response_code = 204
uri = /services
|
| 183 | 2023-05-24T20:19:05.392Z | INFO | SledAgent (RSS): Waiting for rack time synchronization
|
| 184 | 2023-05-24T20:19:05.392Z | INFO | SledAgent (RSS): Checking time synchronization for [fd00:1122:3344:101::1]:12345...
|
| 185 | 2023-05-24T20:19:05.392Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:46381
|
| 186 | 2023-05-24T20:19:05.392Z | INFO | SledAgent (ServiceManager): Configured to skip timesync checks
|
| 187 | 2023-05-24T20:19:05.392Z | INFO | SledAgent (ServiceManager): Setting boot time to 1684959545.392503s
|
| 188 | 2023-05-24T20:19:05.397Z | INFO | SledAgent (ServiceManager): Updated /zone/oxz_internal_dns/root/var/adm/utmpx
|
| 189 | 2023-05-24T20:19:05.402Z | INFO | SledAgent (ServiceManager): Updated /zone/oxz_internal_dns/root/var/adm/wtmpx
|
| 190 | 2023-05-24T20:19:05.407Z | INFO | SledAgent (ServiceManager): Updated /zone/oxz_ntp/root/var/adm/utmpx
|
| 191 | 2023-05-24T20:19:05.412Z | INFO | SledAgent (ServiceManager): Updated /zone/oxz_ntp/root/var/adm/wtmpx
|
| 192 | 2023-05-24T20:19:05.418Z | INFO | SledAgent (ServiceManager): Updated /var/adm/utmpx
|
| 193 | 2023-05-24T20:19:05.423Z | INFO | SledAgent (ServiceManager): Updated /var/adm/wtmpx
|
| 194 | 2023-05-24T20:19:05.423Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = GET
remote_addr = [fd00:1122:3344:101::1]:46381
req_id = 04ab5fa2-8267-4d5a-aa70-02a09bdc2928
response_code = 200
uri = /timesync
|
| 195 | 2023-05-24T20:19:05.423Z | INFO | SledAgent (RSS): Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, skew: 0.0, correction: 0.0 }
|
| 196 | 2023-05-24T20:19:05.423Z | INFO | SledAgent (RSS): sending dataset requests...
|
| 197 | 2023-05-24T20:19:05.423Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 198 | 2023-05-24T20:19:05.423Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:50743
|
| 199 | 2023-05-24T20:19:05.424Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 200 | 2023-05-24T20:19:05.466Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 201 | 2023-05-24T20:19:05.466Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 202 | 2023-05-24T20:19:05.481Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 203 | 2023-05-24T20:19:05.499Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 204 | 2023-05-24T20:19:06.579Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 51.256340556s
|
| 205 | 2023-05-24T20:19:06.624Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 51.300691182s
|
| 206 | 2023-05-24T20:19:06.667Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 51.34400231s
|
| 207 | 2023-05-24T20:19:06.757Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 51.434268274s
|
| 208 | 2023-05-24T20:19:07.312Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 52.173191068s
|
| 209 | 2023-05-24T20:19:07.593Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 52.269776073s
|
| 210 | 2023-05-24T20:19:07.720Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 52.580862341s
|
| 211 | 2023-05-24T20:19:07.803Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService5"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 212 | 2023-05-24T20:19:07.803Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 213 | 2023-05-24T20:19:07.884Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 52.561606582s
|
| 214 | 2023-05-24T20:19:08.084Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 52.230170914s
|
| 215 | 2023-05-24T20:19:08.109Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 52.970184438s
|
| 216 | 2023-05-24T20:19:08.180Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 53.04035001s
|
| 217 | 2023-05-24T20:19:09.397Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 218 | 2023-05-24T20:19:12.246Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService5] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService5 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 983f90b5-3d8e-404a-b42f-6355396d08d8
response_code = 500
uri = /filesystem
|
| 219 | 2023-05-24T20:19:12.249Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "983f90b5-3d8e-404a-b42f-6355396d08d8", "content-length": "124", "date": "Wed, 24 May 2023 20:19:11 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "983f90b5-3d8e-404a-b42f-6355396d08d8" }
|
| 220 | 2023-05-24T20:19:12.311Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 221 | 2023-05-24T20:19:12.312Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 222 | 2023-05-24T20:19:12.329Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 223 | 2023-05-24T20:19:12.329Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 224 | 2023-05-24T20:19:12.342Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 225 | 2023-05-24T20:19:12.342Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 226 | 2023-05-24T20:19:12.358Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 3f7e744e-7a41-401c-81d7-ce12021dcc70
response_code = 500
uri = /filesystem
|
| 227 | 2023-05-24T20:19:12.361Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3f7e744e-7a41-401c-81d7-ce12021dcc70", "content-length": "124", "date": "Wed, 24 May 2023 20:19:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3f7e744e-7a41-401c-81d7-ce12021dcc70" }
|
| 228 | 2023-05-24T20:19:12.423Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 229 | 2023-05-24T20:19:12.424Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 230 | 2023-05-24T20:19:12.440Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 231 | 2023-05-24T20:19:12.440Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 232 | 2023-05-24T20:19:12.455Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 233 | 2023-05-24T20:19:12.455Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 234 | 2023-05-24T20:19:12.472Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = e8c16944-9bee-4541-96cd-dac1d532fc35
response_code = 500
uri = /filesystem
|
| 235 | 2023-05-24T20:19:12.475Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e8c16944-9bee-4541-96cd-dac1d532fc35", "content-length": "124", "date": "Wed, 24 May 2023 20:19:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e8c16944-9bee-4541-96cd-dac1d532fc35" }
|
| 236 | 2023-05-24T20:19:12.582Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 237 | 2023-05-24T20:19:12.583Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 238 | 2023-05-24T20:19:12.598Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 239 | 2023-05-24T20:19:12.598Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 240 | 2023-05-24T20:19:12.613Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 241 | 2023-05-24T20:19:12.613Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 242 | 2023-05-24T20:19:12.630Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = b5c368b0-2204-4c23-89dd-63efbaa4e966
response_code = 500
uri = /filesystem
|
| 243 | 2023-05-24T20:19:12.633Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b5c368b0-2204-4c23-89dd-63efbaa4e966", "content-length": "124", "date": "Wed, 24 May 2023 20:19:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b5c368b0-2204-4c23-89dd-63efbaa4e966" }
|
| 244 | 2023-05-24T20:19:12.806Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 245 | 2023-05-24T20:19:12.806Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 246 | 2023-05-24T20:19:12.822Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 247 | 2023-05-24T20:19:12.822Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 248 | 2023-05-24T20:19:12.838Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 249 | 2023-05-24T20:19:12.838Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 250 | 2023-05-24T20:19:12.856Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 9ad7f7ea-a3fe-46a3-a7f9-ce3a4fcdb140
response_code = 500
uri = /filesystem
|
| 251 | 2023-05-24T20:19:12.858Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "9ad7f7ea-a3fe-46a3-a7f9-ce3a4fcdb140", "content-length": "124", "date": "Wed, 24 May 2023 20:19:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "9ad7f7ea-a3fe-46a3-a7f9-ce3a4fcdb140" }
|
| 252 | 2023-05-24T20:19:13.008Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 253 | 2023-05-24T20:19:13.008Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 254 | 2023-05-24T20:19:13.024Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 255 | 2023-05-24T20:19:13.024Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 256 | 2023-05-24T20:19:13.039Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 257 | 2023-05-24T20:19:13.039Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 258 | 2023-05-24T20:19:13.056Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = d738d33a-7a89-453c-8874-a9ffdfbdcf2f
response_code = 500
uri = /filesystem
|
| 259 | 2023-05-24T20:19:13.059Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d738d33a-7a89-453c-8874-a9ffdfbdcf2f", "content-length": "124", "date": "Wed, 24 May 2023 20:19:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d738d33a-7a89-453c-8874-a9ffdfbdcf2f" }
|
| 260 | 2023-05-24T20:19:13.306Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 261 | 2023-05-24T20:19:13.307Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 262 | 2023-05-24T20:19:13.324Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 263 | 2023-05-24T20:19:13.324Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 264 | 2023-05-24T20:19:13.339Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Down
|
| 265 | 2023-05-24T20:19:13.339Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 266 | 2023-05-24T20:19:13.358Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = c8683b8c-4059-4ab7-b010-9dd98d668a7c
response_code = 500
uri = /filesystem
|
| 267 | 2023-05-24T20:19:13.361Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c8683b8c-4059-4ab7-b010-9dd98d668a7c", "content-length": "124", "date": "Wed, 24 May 2023 20:19:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c8683b8c-4059-4ab7-b010-9dd98d668a7c" }
|
| 268 | 2023-05-24T20:19:13.661Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 269 | 2023-05-24T20:19:13.661Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 270 | 2023-05-24T20:19:13.676Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 271 | 2023-05-24T20:19:13.676Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 272 | 2023-05-24T20:19:13.679Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 273 | 2023-05-24T20:19:13.679Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 274 | 2023-05-24T20:19:13.693Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 275 | 2023-05-24T20:19:13.712Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 276 | 2023-05-24T20:19:15.999Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService12"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 277 | 2023-05-24T20:19:15.999Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 278 | 2023-05-24T20:19:20.605Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService12] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService12 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = d753ef5c-4257-4527-a9d5-aab278b085ca
response_code = 500
uri = /filesystem
|
| 279 | 2023-05-24T20:19:20.610Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d753ef5c-4257-4527-a9d5-aab278b085ca", "content-length": "124", "date": "Wed, 24 May 2023 20:19:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d753ef5c-4257-4527-a9d5-aab278b085ca" }
|
| 280 | 2023-05-24T20:19:20.965Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 281 | 2023-05-24T20:19:20.966Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 282 | 2023-05-24T20:19:20.981Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 283 | 2023-05-24T20:19:20.981Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 284 | 2023-05-24T20:19:20.995Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 285 | 2023-05-24T20:19:20.995Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 286 | 2023-05-24T20:19:21.012Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 25f96b34-8bc5-4137-b373-6ede53407b36
response_code = 500
uri = /filesystem
|
| 287 | 2023-05-24T20:19:21.014Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "25f96b34-8bc5-4137-b373-6ede53407b36", "content-length": "124", "date": "Wed, 24 May 2023 20:19:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "25f96b34-8bc5-4137-b373-6ede53407b36" }
|
| 288 | 2023-05-24T20:19:21.400Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 289 | 2023-05-24T20:19:21.400Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 290 | 2023-05-24T20:19:21.416Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 291 | 2023-05-24T20:19:21.416Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 292 | 2023-05-24T20:19:21.430Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 293 | 2023-05-24T20:19:21.430Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 294 | 2023-05-24T20:19:21.447Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = a55827e6-3059-45e4-af2c-459ecf847877
response_code = 500
uri = /filesystem
|
| 295 | 2023-05-24T20:19:21.449Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a55827e6-3059-45e4-af2c-459ecf847877", "content-length": "124", "date": "Wed, 24 May 2023 20:19:20 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a55827e6-3059-45e4-af2c-459ecf847877" }
|
| 296 | 2023-05-24T20:19:21.922Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 297 | 2023-05-24T20:19:21.922Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 298 | 2023-05-24T20:19:21.940Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 299 | 2023-05-24T20:19:21.940Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 300 | 2023-05-24T20:19:21.955Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Incomplete
|
| 301 | 2023-05-24T20:19:21.955Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 302 | 2023-05-24T20:19:22.053Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 303 | 2023-05-24T20:19:22.053Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 304 | 2023-05-24T20:19:22.058Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured\noxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = efa9b23d-5e5f-4fd5-a756-72997df5068e
response_code = 500
uri = /filesystem
|
| 305 | 2023-05-24T20:19:22.065Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "efa9b23d-5e5f-4fd5-a756-72997df5068e", "content-length": "124", "date": "Wed, 24 May 2023 20:19:22 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "efa9b23d-5e5f-4fd5-a756-72997df5068e" }
|
| 306 | 2023-05-24T20:19:22.693Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 307 | 2023-05-24T20:19:22.693Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 308 | 2023-05-24T20:19:22.711Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 309 | 2023-05-24T20:19:22.711Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 310 | 2023-05-24T20:19:22.725Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 311 | 2023-05-24T20:19:22.742Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 312 | 2023-05-24T20:19:23.269Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 67.946084988s
|
| 313 | 2023-05-24T20:19:23.630Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 68.306866787s
|
| 314 | 2023-05-24T20:19:23.756Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 68.432989288s
|
| 315 | 2023-05-24T20:19:23.787Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 68.464060385s
|
| 316 | 2023-05-24T20:19:23.869Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 68.729893643s
|
| 317 | 2023-05-24T20:19:24.300Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 69.16108589s
|
| 318 | 2023-05-24T20:19:24.399Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 68.544476953s
|
| 319 | 2023-05-24T20:19:24.726Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 69.402808558s
|
| 320 | 2023-05-24T20:19:24.923Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService16"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 321 | 2023-05-24T20:19:24.923Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 322 | 2023-05-24T20:19:25.428Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 70.105206978s
|
| 323 | 2023-05-24T20:19:25.680Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 70.540868174s
|
| 324 | 2023-05-24T20:19:26.044Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 70.905162927s
|
| 325 | 2023-05-24T20:19:26.512Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 326 | 2023-05-24T20:19:30.146Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService16] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService16 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = e49c3745-d302-4d40-a570-f3e0f920912e
response_code = 500
uri = /filesystem
|
| 327 | 2023-05-24T20:19:30.149Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e49c3745-d302-4d40-a570-f3e0f920912e", "content-length": "124", "date": "Wed, 24 May 2023 20:19:29 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e49c3745-d302-4d40-a570-f3e0f920912e" }
|
| 328 | 2023-05-24T20:19:30.892Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 329 | 2023-05-24T20:19:30.892Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 330 | 2023-05-24T20:19:30.912Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 331 | 2023-05-24T20:19:30.912Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 332 | 2023-05-24T20:19:30.929Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 333 | 2023-05-24T20:19:30.929Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 334 | 2023-05-24T20:19:30.948Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 757b8d78-5ff4-4707-83d8-7760cc5c2014
response_code = 500
uri = /filesystem
|
| 335 | 2023-05-24T20:19:30.951Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "757b8d78-5ff4-4707-83d8-7760cc5c2014", "content-length": "124", "date": "Wed, 24 May 2023 20:19:30 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "757b8d78-5ff4-4707-83d8-7760cc5c2014" }
|
| 336 | 2023-05-24T20:19:31.641Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 337 | 2023-05-24T20:19:31.641Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 338 | 2023-05-24T20:19:32.081Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 339 | 2023-05-24T20:19:32.081Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 340 | 2023-05-24T20:19:32.099Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 341 | 2023-05-24T20:19:32.099Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 342 | 2023-05-24T20:19:32.115Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 343 | 2023-05-24T20:19:32.134Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 344 | 2023-05-24T20:19:34.326Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService18"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 345 | 2023-05-24T20:19:34.327Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 346 | 2023-05-24T20:19:38.934Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService18] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService18 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 1e80d05a-8fab-4272-a45d-b2a4dca03cc1
response_code = 500
uri = /filesystem
|
| 347 | 2023-05-24T20:19:38.939Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1e80d05a-8fab-4272-a45d-b2a4dca03cc1", "content-length": "124", "date": "Wed, 24 May 2023 20:19:38 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1e80d05a-8fab-4272-a45d-b2a4dca03cc1" }
|
| 348 | 2023-05-24T20:19:40.167Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 84.844275811s
|
| 349 | 2023-05-24T20:19:40.359Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 350 | 2023-05-24T20:19:40.359Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 351 | 2023-05-24T20:19:40.430Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 352 | 2023-05-24T20:19:40.430Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 353 | 2023-05-24T20:19:40.448Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 354 | 2023-05-24T20:19:40.448Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 355 | 2023-05-24T20:19:40.462Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 356 | 2023-05-24T20:19:40.480Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 357 | 2023-05-24T20:19:40.536Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 85.2131995s
|
| 358 | 2023-05-24T20:19:41.156Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 85.83366605s
|
| 359 | 2023-05-24T20:19:41.465Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 86.141858021s
|
| 360 | 2023-05-24T20:19:41.512Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 86.372737676s
|
| 361 | 2023-05-24T20:19:41.513Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 85.659031511s
|
| 362 | 2023-05-24T20:19:42.062Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 86.739487805s
|
| 363 | 2023-05-24T20:19:42.096Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 86.773645205s
|
| 364 | 2023-05-24T20:19:42.612Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService19"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 365 | 2023-05-24T20:19:42.612Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 366 | 2023-05-24T20:19:42.786Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 87.647065977s
|
| 367 | 2023-05-24T20:19:43.040Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 87.900911861s
|
| 368 | 2023-05-24T20:19:43.267Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 369 | 2023-05-24T20:19:44.694Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 89.555105145s
|
| 370 | 2023-05-24T20:19:47.184Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService19] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService19 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 78854f79-3d01-49f9-a540-2b08a782a7b8
response_code = 500
uri = /filesystem
|
| 371 | 2023-05-24T20:19:47.187Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "78854f79-3d01-49f9-a540-2b08a782a7b8", "content-length": "124", "date": "Wed, 24 May 2023 20:19:47 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "78854f79-3d01-49f9-a540-2b08a782a7b8" }
|
| 372 | 2023-05-24T20:19:48.586Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 373 | 2023-05-24T20:19:48.586Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 374 | 2023-05-24T20:19:49.094Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 375 | 2023-05-24T20:19:49.094Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 376 | 2023-05-24T20:19:49.113Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 377 | 2023-05-24T20:19:49.113Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 378 | 2023-05-24T20:19:49.127Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 379 | 2023-05-24T20:19:49.146Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 380 | 2023-05-24T20:19:51.268Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService20"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 381 | 2023-05-24T20:19:51.269Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 382 | 2023-05-24T20:19:55.648Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService20] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService20 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 469e4898-8975-4859-932a-5b97dd2430be
response_code = 500
uri = /filesystem
|
| 383 | 2023-05-24T20:19:55.652Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "469e4898-8975-4859-932a-5b97dd2430be", "content-length": "124", "date": "Wed, 24 May 2023 20:19:54 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "469e4898-8975-4859-932a-5b97dd2430be" }
|
| 384 | 2023-05-24T20:19:57.050Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 385 | 2023-05-24T20:19:57.050Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 386 | 2023-05-24T20:19:57.451Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }
|
| 387 | 2023-05-24T20:19:57.452Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 388 | 2023-05-24T20:19:57.469Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 389 | 2023-05-24T20:19:57.469Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb does not yet exist
|
| 390 | 2023-05-24T20:19:57.484Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 391 | 2023-05-24T20:19:57.503Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 392 | 2023-05-24T20:19:57.636Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 102.312796184s
|
| 393 | 2023-05-24T20:19:58.065Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 102.742255004s
|
| 394 | 2023-05-24T20:19:58.268Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 102.414322591s
|
| 395 | 2023-05-24T20:19:58.759Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 103.435747774s
|
| 396 | 2023-05-24T20:19:59.622Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 104.483052923s
|
| 397 | 2023-05-24T20:19:59.623Z | INFO | SledAgent (ServiceManager): Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/cockroachdb">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService21"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::6"/>
<propval type="astring" name="listen_port" value="32221"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 398 | 2023-05-24T20:19:59.623Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 399 | 2023-05-24T20:19:59.820Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 104.496982329s
|
| 400 | 2023-05-24T20:20:00.447Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 105.124641053s
|
| 401 | 2023-05-24T20:20:00.597Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 105.273836886s
|
| 402 | 2023-05-24T20:20:01.193Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 106.053842698s
|
| 403 | 2023-05-24T20:20:01.839Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 106.699978341s
|
| 404 | 2023-05-24T20:20:02.756Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 107.617421639s
|
| 405 | 2023-05-24T20:20:03.337Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 406 | 2023-05-24T20:20:03.819Z | INFO | SledAgent (ServiceManager): Formatting CRDB
|
| 407 | 2023-05-24T20:20:03.997Z | WARN | SledAgent (ServiceManager): failed to format CRDB
error = RunCommandError { zone: "oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03", err: CommandFailure(CommandFailureInfo { command: "/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::6]:32221 --file /opt/oxide/cockroachdb/sql/dbwipe.sql", status: ExitStatus(unix_wait_status(256)), stdout: "", stderr: "ERROR: cannot dial server.\\nIs the server running?\\nIf the server is running, check --host client-side and --advertise server-side.\\n\\ndial tcp [fd00:1122:3344:101::6]:32221: connect: connection refused\\nFailed running \\"sql\\"\\n" }) }
|
| 408 | 2023-05-24T20:20:04.025Z | INFO | SledAgent (ServiceManager): Formatting CRDB
|
| 409 | 2023-05-24T20:20:04.175Z | WARN | SledAgent (ServiceManager): failed to format CRDB
error = RunCommandError { zone: "oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03", err: CommandFailure(CommandFailureInfo { command: "/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::6]:32221 --file /opt/oxide/cockroachdb/sql/dbwipe.sql", status: ExitStatus(unix_wait_status(256)), stdout: "", stderr: "ERROR: cannot dial server.\\nIs the server running?\\nIf the server is running, check --host client-side and --advertise server-side.\\n\\ndial tcp [fd00:1122:3344:101::6]:32221: connect: connection refused\\nFailed running \\"sql\\"\\n" }) }
|
| 410 | 2023-05-24T20:20:04.307Z | INFO | SledAgent (ServiceManager): Formatting CRDB
|
| 411 | 2023-05-24T20:20:04.459Z | WARN | SledAgent (ServiceManager): failed to format CRDB
error = RunCommandError { zone: "oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03", err: CommandFailure(CommandFailureInfo { command: "/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::6]:32221 --file /opt/oxide/cockroachdb/sql/dbwipe.sql", status: ExitStatus(unix_wait_status(256)), stdout: "", stderr: "ERROR: cannot dial server.\\nIs the server running?\\nIf the server is running, check --host client-side and --advertise server-side.\\n\\ndial tcp [fd00:1122:3344:101::6]:32221: connect: connection refused\\nFailed running \\"sql\\"\\n" }) }
|
| 412 | 2023-05-24T20:20:04.621Z | INFO | SledAgent (ServiceManager): Formatting CRDB
|
| 413 | 2023-05-24T20:20:12.782Z | INFO | SledAgent (ServiceManager): Formatting CRDB - Completed
|
| 414 | 2023-05-24T20:20:12.783Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 1d988b97-e460-44a0-9513-3ce2a2609ff6
response_code = 204
uri = /filesystem
|
| 415 | 2023-05-24T20:20:12.784Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 1f2e9de8-4e2a-4410-81a7-1f606e9e8c47, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::7]:8123 }
|
| 416 | 2023-05-24T20:20:12.784Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 1f2e9de8-4e2a-4410-81a7-1f606e9e8c47, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Clickhouse, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 417 | 2023-05-24T20:20:12.841Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 418 | 2023-05-24T20:20:12.841Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 419 | 2023-05-24T20:20:12.841Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 420 | 2023-05-24T20:20:12.841Z | INFO | SledAgent (ServiceManager): Service zone clickhouse does not yet exist
|
| 421 | 2023-05-24T20:20:12.858Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_clickhouse_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 422 | 2023-05-24T20:20:12.876Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_clickhouse_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 423 | 2023-05-24T20:20:15.019Z | INFO | SledAgent (ServiceManager): Profile for oxz_clickhouse_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/clickhouse">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService22"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::7"/>
<propval type="astring" name="listen_port" value="8123"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 424 | 2023-05-24T20:20:15.019Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_clickhouse_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 425 | 2023-05-24T20:20:15.272Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 119.949430479s
|
| 426 | 2023-05-24T20:20:16.466Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 121.142953592s
|
| 427 | 2023-05-24T20:20:16.846Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 121.522691402s
|
| 428 | 2023-05-24T20:20:18.328Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 123.189055223s
|
| 429 | 2023-05-24T20:20:18.338Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 122.484444763s
|
| 430 | 2023-05-24T20:20:19.002Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 123.678884371s
|
| 431 | 2023-05-24T20:20:19.199Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 123.875769079s
|
| 432 | 2023-05-24T20:20:19.199Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = f5ef5de3-f332-4922-a12b-e2096e0f1675
response_code = 204
uri = /filesystem
|
| 433 | 2023-05-24T20:20:19.199Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 434 | 2023-05-24T20:20:19.199Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 435 | 2023-05-24T20:20:19.260Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 436 | 2023-05-24T20:20:19.260Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 437 | 2023-05-24T20:20:19.260Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 438 | 2023-05-24T20:20:19.260Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 439 | 2023-05-24T20:20:19.260Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 440 | 2023-05-24T20:20:19.260Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 441 | 2023-05-24T20:20:19.277Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 442 | 2023-05-24T20:20:19.298Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 443 | 2023-05-24T20:20:20.611Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService23"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="def968e0-1ba4-4906-bf26-b448f9d4820d"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 444 | 2023-05-24T20:20:20.611Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 445 | 2023-05-24T20:20:24.430Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 446 | 2023-05-24T20:20:25.380Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService23] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService23 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = e3e037c0-ccd5-4e3f-ac98-5738456f42be
response_code = 500
uri = /filesystem
|
| 447 | 2023-05-24T20:20:25.383Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e3e037c0-ccd5-4e3f-ac98-5738456f42be", "content-length": "124", "date": "Wed, 24 May 2023 20:20:24 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e3e037c0-ccd5-4e3f-ac98-5738456f42be" }
|
| 448 | 2023-05-24T20:20:25.485Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 449 | 2023-05-24T20:20:25.486Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 450 | 2023-05-24T20:20:25.503Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 451 | 2023-05-24T20:20:25.503Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 452 | 2023-05-24T20:20:25.503Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 453 | 2023-05-24T20:20:25.503Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 454 | 2023-05-24T20:20:25.503Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 455 | 2023-05-24T20:20:25.503Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 456 | 2023-05-24T20:20:25.517Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 457 | 2023-05-24T20:20:25.518Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 458 | 2023-05-24T20:20:25.536Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 429047b6-e91e-4c18-b26d-7224d70ba5d2
response_code = 500
uri = /filesystem
|
| 459 | 2023-05-24T20:20:25.539Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "429047b6-e91e-4c18-b26d-7224d70ba5d2", "content-length": "124", "date": "Wed, 24 May 2023 20:20:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "429047b6-e91e-4c18-b26d-7224d70ba5d2" }
|
| 460 | 2023-05-24T20:20:25.686Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 461 | 2023-05-24T20:20:25.687Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 462 | 2023-05-24T20:20:25.704Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 463 | 2023-05-24T20:20:25.704Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 464 | 2023-05-24T20:20:25.704Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 465 | 2023-05-24T20:20:25.704Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 466 | 2023-05-24T20:20:25.704Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 467 | 2023-05-24T20:20:25.704Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 468 | 2023-05-24T20:20:25.719Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 469 | 2023-05-24T20:20:25.719Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 470 | 2023-05-24T20:20:25.737Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 00be8081-b8ba-4595-a6ee-985af134cb8d
response_code = 500
uri = /filesystem
|
| 471 | 2023-05-24T20:20:25.740Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "00be8081-b8ba-4595-a6ee-985af134cb8d", "content-length": "124", "date": "Wed, 24 May 2023 20:20:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "00be8081-b8ba-4595-a6ee-985af134cb8d" }
|
| 472 | 2023-05-24T20:20:25.874Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 473 | 2023-05-24T20:20:25.874Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 474 | 2023-05-24T20:20:25.891Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 475 | 2023-05-24T20:20:25.891Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 476 | 2023-05-24T20:20:25.891Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 477 | 2023-05-24T20:20:25.891Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 478 | 2023-05-24T20:20:25.891Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 479 | 2023-05-24T20:20:25.891Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 480 | 2023-05-24T20:20:25.907Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 481 | 2023-05-24T20:20:25.907Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 482 | 2023-05-24T20:20:25.925Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = ef21ae8c-10f4-476c-8546-3f2b8d096803
response_code = 500
uri = /filesystem
|
| 483 | 2023-05-24T20:20:25.927Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "ef21ae8c-10f4-476c-8546-3f2b8d096803", "content-length": "124", "date": "Wed, 24 May 2023 20:20:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "ef21ae8c-10f4-476c-8546-3f2b8d096803" }
|
| 484 | 2023-05-24T20:20:26.063Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 485 | 2023-05-24T20:20:26.063Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 486 | 2023-05-24T20:20:26.080Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 487 | 2023-05-24T20:20:26.080Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 488 | 2023-05-24T20:20:26.080Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 489 | 2023-05-24T20:20:26.080Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 490 | 2023-05-24T20:20:26.080Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 491 | 2023-05-24T20:20:26.080Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 492 | 2023-05-24T20:20:26.096Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 493 | 2023-05-24T20:20:26.096Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 494 | 2023-05-24T20:20:26.115Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 67350fbc-b142-4761-9304-b54df2394bdf
response_code = 500
uri = /filesystem
|
| 495 | 2023-05-24T20:20:26.118Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "67350fbc-b142-4761-9304-b54df2394bdf", "content-length": "124", "date": "Wed, 24 May 2023 20:20:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "67350fbc-b142-4761-9304-b54df2394bdf" }
|
| 496 | 2023-05-24T20:20:26.328Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 497 | 2023-05-24T20:20:26.328Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 498 | 2023-05-24T20:20:26.346Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 499 | 2023-05-24T20:20:26.346Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 500 | 2023-05-24T20:20:26.346Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 501 | 2023-05-24T20:20:26.346Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 502 | 2023-05-24T20:20:26.346Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 503 | 2023-05-24T20:20:26.346Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 504 | 2023-05-24T20:20:26.362Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 505 | 2023-05-24T20:20:26.362Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 506 | 2023-05-24T20:20:26.382Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 8e4eaaaa-cb70-4eb4-84e9-09c5776dbc22
response_code = 500
uri = /filesystem
|
| 507 | 2023-05-24T20:20:26.385Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8e4eaaaa-cb70-4eb4-84e9-09c5776dbc22", "content-length": "124", "date": "Wed, 24 May 2023 20:20:25 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8e4eaaaa-cb70-4eb4-84e9-09c5776dbc22" }
|
| 508 | 2023-05-24T20:20:26.589Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 509 | 2023-05-24T20:20:26.590Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 510 | 2023-05-24T20:20:26.609Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 511 | 2023-05-24T20:20:26.609Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 512 | 2023-05-24T20:20:26.609Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 513 | 2023-05-24T20:20:26.609Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 514 | 2023-05-24T20:20:26.609Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 515 | 2023-05-24T20:20:26.609Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 516 | 2023-05-24T20:20:26.624Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Incomplete
|
| 517 | 2023-05-24T20:20:26.624Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 518 | 2023-05-24T20:20:26.786Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 519 | 2023-05-24T20:20:26.786Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 520 | 2023-05-24T20:20:26.791Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured\noxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 3c726e74-c9eb-4da6-9e55-5d6e1466bf5b
response_code = 500
uri = /filesystem
|
| 521 | 2023-05-24T20:20:26.798Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3c726e74-c9eb-4da6-9e55-5d6e1466bf5b", "content-length": "124", "date": "Wed, 24 May 2023 20:20:26 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3c726e74-c9eb-4da6-9e55-5d6e1466bf5b" }
|
| 522 | 2023-05-24T20:20:27.109Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 523 | 2023-05-24T20:20:27.110Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 524 | 2023-05-24T20:20:27.128Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 525 | 2023-05-24T20:20:27.128Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 526 | 2023-05-24T20:20:27.128Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 527 | 2023-05-24T20:20:27.128Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 528 | 2023-05-24T20:20:27.128Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 529 | 2023-05-24T20:20:27.128Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 530 | 2023-05-24T20:20:27.144Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 531 | 2023-05-24T20:20:27.163Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 532 | 2023-05-24T20:20:27.846Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 132.523307332s
|
| 533 | 2023-05-24T20:20:27.846Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 132.70718534s
|
| 534 | 2023-05-24T20:20:27.847Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 132.707650363s
|
| 535 | 2023-05-24T20:20:27.847Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 132.707592291s
|
| 536 | 2023-05-24T20:20:28.568Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService30"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="6a40918e-faf7-41c2-9d9a-5940ca854305"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 537 | 2023-05-24T20:20:28.568Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 538 | 2023-05-24T20:20:33.357Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService30] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService30 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = b1ec6d87-32b4-4f42-9805-fc0fc2a3290a
response_code = 500
uri = /filesystem
|
| 539 | 2023-05-24T20:20:33.360Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b1ec6d87-32b4-4f42-9805-fc0fc2a3290a", "content-length": "124", "date": "Wed, 24 May 2023 20:20:33 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b1ec6d87-32b4-4f42-9805-fc0fc2a3290a" }
|
| 540 | 2023-05-24T20:20:33.865Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 541 | 2023-05-24T20:20:33.865Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 542 | 2023-05-24T20:20:33.884Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 543 | 2023-05-24T20:20:33.884Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 544 | 2023-05-24T20:20:33.884Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 545 | 2023-05-24T20:20:33.884Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 546 | 2023-05-24T20:20:33.884Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 547 | 2023-05-24T20:20:33.884Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 548 | 2023-05-24T20:20:33.900Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 549 | 2023-05-24T20:20:33.900Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 550 | 2023-05-24T20:20:33.920Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 72eee2a2-9b37-494d-bca3-d69b040daecd
response_code = 500
uri = /filesystem
|
| 551 | 2023-05-24T20:20:33.923Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "72eee2a2-9b37-494d-bca3-d69b040daecd", "content-length": "124", "date": "Wed, 24 May 2023 20:20:33 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "72eee2a2-9b37-494d-bca3-d69b040daecd" }
|
| 552 | 2023-05-24T20:20:34.403Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 553 | 2023-05-24T20:20:34.403Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 554 | 2023-05-24T20:20:34.421Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 555 | 2023-05-24T20:20:34.421Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 556 | 2023-05-24T20:20:34.421Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 557 | 2023-05-24T20:20:34.421Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 558 | 2023-05-24T20:20:34.421Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 559 | 2023-05-24T20:20:34.421Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 560 | 2023-05-24T20:20:34.438Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Down
|
| 561 | 2023-05-24T20:20:34.438Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 562 | 2023-05-24T20:20:34.458Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = a671e7ea-37f4-4233-87f4-ffdf7f05397b
response_code = 500
uri = /filesystem
|
| 563 | 2023-05-24T20:20:34.461Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a671e7ea-37f4-4233-87f4-ffdf7f05397b", "content-length": "124", "date": "Wed, 24 May 2023 20:20:33 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a671e7ea-37f4-4233-87f4-ffdf7f05397b" }
|
| 564 | 2023-05-24T20:20:34.768Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 565 | 2023-05-24T20:20:34.768Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 566 | 2023-05-24T20:20:35.224Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 567 | 2023-05-24T20:20:35.225Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 568 | 2023-05-24T20:20:35.244Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 569 | 2023-05-24T20:20:35.244Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 570 | 2023-05-24T20:20:35.244Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 571 | 2023-05-24T20:20:35.244Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 572 | 2023-05-24T20:20:35.244Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 573 | 2023-05-24T20:20:35.244Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 574 | 2023-05-24T20:20:35.260Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 575 | 2023-05-24T20:20:35.278Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 576 | 2023-05-24T20:20:35.354Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 140.031334738s
|
| 577 | 2023-05-24T20:20:36.382Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 141.05907192s
|
| 578 | 2023-05-24T20:20:36.441Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 141.11838635s
|
| 579 | 2023-05-24T20:20:36.598Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService33"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="c39ce1b8-8eb8-41b8-ab1e-97de361a8c5b"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 580 | 2023-05-24T20:20:36.598Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 581 | 2023-05-24T20:20:36.638Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 141.315104734s
|
| 582 | 2023-05-24T20:20:37.345Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 142.206233212s
|
| 583 | 2023-05-24T20:20:39.433Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 143.5787651s
|
| 584 | 2023-05-24T20:20:39.982Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 144.659338237s
|
| 585 | 2023-05-24T20:20:41.731Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService33] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService33 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = a5ebae1f-d555-4cd8-890e-233fa8eed1c1
response_code = 500
uri = /filesystem
|
| 586 | 2023-05-24T20:20:41.734Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "a5ebae1f-d555-4cd8-890e-233fa8eed1c1", "content-length": "124", "date": "Wed, 24 May 2023 20:20:41 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "a5ebae1f-d555-4cd8-890e-233fa8eed1c1" }
|
| 587 | 2023-05-24T20:20:42.483Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 588 | 2023-05-24T20:20:42.483Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 589 | 2023-05-24T20:20:42.501Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 590 | 2023-05-24T20:20:42.501Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 591 | 2023-05-24T20:20:42.501Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 592 | 2023-05-24T20:20:42.501Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 593 | 2023-05-24T20:20:42.501Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 594 | 2023-05-24T20:20:42.501Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 595 | 2023-05-24T20:20:42.517Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 596 | 2023-05-24T20:20:42.517Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 597 | 2023-05-24T20:20:42.537Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 1420e94f-13d3-4ed1-9416-ab6502f01002
response_code = 500
uri = /filesystem
|
| 598 | 2023-05-24T20:20:42.540Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1420e94f-13d3-4ed1-9416-ab6502f01002", "content-length": "124", "date": "Wed, 24 May 2023 20:20:42 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1420e94f-13d3-4ed1-9416-ab6502f01002" }
|
| 599 | 2023-05-24T20:20:43.155Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 600 | 2023-05-24T20:20:43.155Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 601 | 2023-05-24T20:20:43.360Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 602 | 2023-05-24T20:20:43.361Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 603 | 2023-05-24T20:20:43.379Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 604 | 2023-05-24T20:20:43.379Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 605 | 2023-05-24T20:20:43.379Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 606 | 2023-05-24T20:20:43.379Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 607 | 2023-05-24T20:20:43.379Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 608 | 2023-05-24T20:20:43.379Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 609 | 2023-05-24T20:20:43.395Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 610 | 2023-05-24T20:20:43.413Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 611 | 2023-05-24T20:20:44.736Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService35"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="d5015eac-6cd1-405e-8084-2ac713a36d2d"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 612 | 2023-05-24T20:20:44.736Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 613 | 2023-05-24T20:20:46.279Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 614 | 2023-05-24T20:20:47.526Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 152.203457578s
|
| 615 | 2023-05-24T20:20:48.079Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 152.940183792s
|
| 616 | 2023-05-24T20:20:48.141Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 153.002260546s
|
| 617 | 2023-05-24T20:20:48.455Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 153.316020756s
|
| 618 | 2023-05-24T20:20:49.866Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService35] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService35 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 53d745f5-e158-4e0b-906f-28b74ce2edb7
response_code = 500
uri = /filesystem
|
| 619 | 2023-05-24T20:20:49.869Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "53d745f5-e158-4e0b-906f-28b74ce2edb7", "content-length": "124", "date": "Wed, 24 May 2023 20:20:49 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "53d745f5-e158-4e0b-906f-28b74ce2edb7" }
|
| 620 | 2023-05-24T20:20:50.893Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 621 | 2023-05-24T20:20:50.893Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 622 | 2023-05-24T20:20:50.911Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 623 | 2023-05-24T20:20:50.911Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 624 | 2023-05-24T20:20:50.911Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 625 | 2023-05-24T20:20:50.911Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 626 | 2023-05-24T20:20:50.911Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 627 | 2023-05-24T20:20:50.911Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 628 | 2023-05-24T20:20:50.928Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Down
|
| 629 | 2023-05-24T20:20:50.928Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 630 | 2023-05-24T20:20:50.948Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 0a555630-00b2-4e73-a75e-cff4e2aef09b
response_code = 500
uri = /filesystem
|
| 631 | 2023-05-24T20:20:50.951Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "0a555630-00b2-4e73-a75e-cff4e2aef09b", "content-length": "124", "date": "Wed, 24 May 2023 20:20:50 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "0a555630-00b2-4e73-a75e-cff4e2aef09b" }
|
| 632 | 2023-05-24T20:20:51.294Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 633 | 2023-05-24T20:20:51.294Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 634 | 2023-05-24T20:20:51.750Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 635 | 2023-05-24T20:20:51.750Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 636 | 2023-05-24T20:20:51.770Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 637 | 2023-05-24T20:20:51.770Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 638 | 2023-05-24T20:20:51.770Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 639 | 2023-05-24T20:20:51.770Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 640 | 2023-05-24T20:20:51.770Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 641 | 2023-05-24T20:20:51.770Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 642 | 2023-05-24T20:20:51.786Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 643 | 2023-05-24T20:20:51.804Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 644 | 2023-05-24T20:20:53.117Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService37"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="448fa1b9-bca7-4986-9a65-a6129c90d483"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 645 | 2023-05-24T20:20:53.117Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 646 | 2023-05-24T20:20:55.359Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 160.219929323s
|
| 647 | 2023-05-24T20:20:55.428Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 160.10559611s
|
| 648 | 2023-05-24T20:20:55.618Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 160.29469327s
|
| 649 | 2023-05-24T20:20:57.118Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 161.794983803s
|
| 650 | 2023-05-24T20:20:57.786Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 162.463157561s
|
| 651 | 2023-05-24T20:20:58.258Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService37] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService37 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 67f55cb5-77b0-4663-945e-3d782d5e7229
response_code = 500
uri = /filesystem
|
| 652 | 2023-05-24T20:20:58.264Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "67f55cb5-77b0-4663-945e-3d782d5e7229", "content-length": "124", "date": "Wed, 24 May 2023 20:20:58 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "67f55cb5-77b0-4663-945e-3d782d5e7229" }
|
| 653 | 2023-05-24T20:20:59.634Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 654 | 2023-05-24T20:20:59.634Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 655 | 2023-05-24T20:20:59.654Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 656 | 2023-05-24T20:20:59.654Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 657 | 2023-05-24T20:20:59.654Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 658 | 2023-05-24T20:20:59.654Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 659 | 2023-05-24T20:20:59.654Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 660 | 2023-05-24T20:20:59.654Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 661 | 2023-05-24T20:20:59.673Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Configured
|
| 662 | 2023-05-24T20:20:59.673Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 663 | 2023-05-24T20:20:59.680Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 664 | 2023-05-24T20:20:59.680Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 665 | 2023-05-24T20:20:59.683Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 666 | 2023-05-24T20:20:59.702Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 667 | 2023-05-24T20:21:01.016Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService38"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="b816375a-a908-4dd9-90e6-218c41f2460c"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 668 | 2023-05-24T20:21:01.016Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 669 | 2023-05-24T20:21:01.282Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 165.427991843s
|
| 670 | 2023-05-24T20:21:02.295Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 166.971714438s
|
| 671 | 2023-05-24T20:21:06.228Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService38] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService38 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 13b7c842-c55b-4a55-afe2-7595dac226c1
response_code = 500
uri = /filesystem
|
| 672 | 2023-05-24T20:21:06.234Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "13b7c842-c55b-4a55-afe2-7595dac226c1", "content-length": "124", "date": "Wed, 24 May 2023 20:21:06 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "13b7c842-c55b-4a55-afe2-7595dac226c1" }
|
| 673 | 2023-05-24T20:21:07.269Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 171.946088478s
|
| 674 | 2023-05-24T20:21:07.658Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 675 | 2023-05-24T20:21:07.658Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 676 | 2023-05-24T20:21:07.718Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 172.579083486s
|
| 677 | 2023-05-24T20:21:07.767Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 678 | 2023-05-24T20:21:07.768Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 679 | 2023-05-24T20:21:07.786Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 680 | 2023-05-24T20:21:07.786Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 681 | 2023-05-24T20:21:07.786Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 682 | 2023-05-24T20:21:07.786Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 683 | 2023-05-24T20:21:07.786Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 684 | 2023-05-24T20:21:07.786Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 685 | 2023-05-24T20:21:07.802Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 686 | 2023-05-24T20:21:07.820Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 687 | 2023-05-24T20:21:09.161Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService39"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="70cba03f-96e3-4f74-9ef4-06d58a3251ef"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 688 | 2023-05-24T20:21:09.161Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 689 | 2023-05-24T20:21:10.255Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 690 | 2023-05-24T20:21:11.128Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 175.989419632s
|
| 691 | 2023-05-24T20:21:11.139Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 175.999557975s
|
| 692 | 2023-05-24T20:21:14.691Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 179.368251179s
|
| 693 | 2023-05-24T20:21:14.870Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService39] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService39 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 88e54b6f-c692-42e3-8241-5e206c427f9d
response_code = 500
uri = /filesystem
|
| 694 | 2023-05-24T20:21:14.873Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "88e54b6f-c692-42e3-8241-5e206c427f9d", "content-length": "124", "date": "Wed, 24 May 2023 20:21:14 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "88e54b6f-c692-42e3-8241-5e206c427f9d" }
|
| 695 | 2023-05-24T20:21:16.297Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 696 | 2023-05-24T20:21:16.297Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 697 | 2023-05-24T20:21:16.339Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 181.199784614s
|
| 698 | 2023-05-24T20:21:16.461Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 181.138479091s
|
| 699 | 2023-05-24T20:21:17.316Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 700 | 2023-05-24T20:21:17.316Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 701 | 2023-05-24T20:21:17.335Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 702 | 2023-05-24T20:21:17.335Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 703 | 2023-05-24T20:21:17.335Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 704 | 2023-05-24T20:21:17.335Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 705 | 2023-05-24T20:21:17.335Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 706 | 2023-05-24T20:21:17.335Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 707 | 2023-05-24T20:21:17.350Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 708 | 2023-05-24T20:21:17.369Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 709 | 2023-05-24T20:21:18.684Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService40"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="bf28318e-8a7f-4d40-aaad-04787bb0ed82"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 710 | 2023-05-24T20:21:18.684Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 711 | 2023-05-24T20:21:18.763Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 183.440403951s
|
| 712 | 2023-05-24T20:21:21.368Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 186.044660131s
|
| 713 | 2023-05-24T20:21:23.349Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 188.026382466s
|
| 714 | 2023-05-24T20:21:24.236Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService40] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService40 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = b3feaedd-78c4-455e-b1f8-736e632d72c6
response_code = 500
uri = /filesystem
|
| 715 | 2023-05-24T20:21:24.239Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b3feaedd-78c4-455e-b1f8-736e632d72c6", "content-length": "124", "date": "Wed, 24 May 2023 20:21:24 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b3feaedd-78c4-455e-b1f8-736e632d72c6" }
|
| 716 | 2023-05-24T20:21:25.258Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 189.403955822s
|
| 717 | 2023-05-24T20:21:25.676Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 718 | 2023-05-24T20:21:25.676Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 719 | 2023-05-24T20:21:26.651Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 720 | 2023-05-24T20:21:26.652Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 721 | 2023-05-24T20:21:26.671Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 722 | 2023-05-24T20:21:26.671Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 723 | 2023-05-24T20:21:26.671Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 724 | 2023-05-24T20:21:26.671Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 725 | 2023-05-24T20:21:26.671Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 726 | 2023-05-24T20:21:26.671Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 727 | 2023-05-24T20:21:26.686Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 728 | 2023-05-24T20:21:26.705Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 729 | 2023-05-24T20:21:26.865Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 191.542529258s
|
| 730 | 2023-05-24T20:21:28.035Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService41"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="b78cba6c-643b-4c2c-aa8c-beac78e34e78"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 731 | 2023-05-24T20:21:28.035Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 732 | 2023-05-24T20:21:28.125Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 192.986221195s
|
| 733 | 2023-05-24T20:21:33.136Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService41] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService41 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 3019ba31-7af9-467a-a2fe-e5be0c5dd916
response_code = 500
uri = /filesystem
|
| 734 | 2023-05-24T20:21:33.139Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3019ba31-7af9-467a-a2fe-e5be0c5dd916", "content-length": "124", "date": "Wed, 24 May 2023 20:21:33 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3019ba31-7af9-467a-a2fe-e5be0c5dd916" }
|
| 735 | 2023-05-24T20:21:33.285Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 736 | 2023-05-24T20:21:33.666Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 198.527248493s
|
| 737 | 2023-05-24T20:21:34.563Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 738 | 2023-05-24T20:21:34.563Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 739 | 2023-05-24T20:21:35.447Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 200.30802311s
|
| 740 | 2023-05-24T20:21:36.476Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 741 | 2023-05-24T20:21:36.476Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 742 | 2023-05-24T20:21:36.496Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 743 | 2023-05-24T20:21:36.496Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 744 | 2023-05-24T20:21:36.496Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 745 | 2023-05-24T20:21:36.496Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 746 | 2023-05-24T20:21:36.496Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 747 | 2023-05-24T20:21:36.496Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 748 | 2023-05-24T20:21:36.512Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 749 | 2023-05-24T20:21:36.532Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 750 | 2023-05-24T20:21:37.148Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 201.825144448s
|
| 751 | 2023-05-24T20:21:37.863Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService42"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="5d46127a-1cd7-4af6-9709-f7a4e3db9e92"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 752 | 2023-05-24T20:21:37.863Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 753 | 2023-05-24T20:21:38.573Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 203.250147911s
|
| 754 | 2023-05-24T20:21:40.915Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 205.77602906s
|
| 755 | 2023-05-24T20:21:43.238Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService42] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService42 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = f8c641c4-ddc8-4861-80fa-c10fc5dac0be
response_code = 500
uri = /filesystem
|
| 756 | 2023-05-24T20:21:43.241Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f8c641c4-ddc8-4861-80fa-c10fc5dac0be", "content-length": "124", "date": "Wed, 24 May 2023 20:21:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f8c641c4-ddc8-4861-80fa-c10fc5dac0be" }
|
| 757 | 2023-05-24T20:21:44.675Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 758 | 2023-05-24T20:21:44.676Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 759 | 2023-05-24T20:21:44.770Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 209.447349533s
|
| 760 | 2023-05-24T20:21:44.903Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 209.57967938s
|
| 761 | 2023-05-24T20:21:46.496Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 211.173343932s
|
| 762 | 2023-05-24T20:21:46.820Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 763 | 2023-05-24T20:21:46.820Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 764 | 2023-05-24T20:21:46.839Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 765 | 2023-05-24T20:21:46.839Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 766 | 2023-05-24T20:21:46.839Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 767 | 2023-05-24T20:21:46.839Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 768 | 2023-05-24T20:21:46.839Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 769 | 2023-05-24T20:21:46.840Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 770 | 2023-05-24T20:21:46.855Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 771 | 2023-05-24T20:21:46.875Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 772 | 2023-05-24T20:21:47.694Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 212.554534105s
|
| 773 | 2023-05-24T20:21:48.198Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService43"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="aadfff4a-e752-43bd-88d4-8abac618c05b"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 774 | 2023-05-24T20:21:48.198Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 775 | 2023-05-24T20:21:48.288Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 212.434046175s
|
| 776 | 2023-05-24T20:21:52.735Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService43] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService43 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 733c3e7a-4deb-4823-9071-e0e0234a9bda
response_code = 500
uri = /filesystem
|
| 777 | 2023-05-24T20:21:52.739Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "733c3e7a-4deb-4823-9071-e0e0234a9bda", "content-length": "124", "date": "Wed, 24 May 2023 20:21:52 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "733c3e7a-4deb-4823-9071-e0e0234a9bda" }
|
| 778 | 2023-05-24T20:21:53.621Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 218.297960651s
|
| 779 | 2023-05-24T20:21:54.161Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 780 | 2023-05-24T20:21:54.161Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 781 | 2023-05-24T20:21:56.534Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 782 | 2023-05-24T20:21:57.961Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 222.638601533s
|
| 783 | 2023-05-24T20:21:58.107Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 784 | 2023-05-24T20:21:58.107Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 785 | 2023-05-24T20:21:58.127Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 786 | 2023-05-24T20:21:58.127Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 787 | 2023-05-24T20:21:58.127Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 788 | 2023-05-24T20:21:58.127Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 789 | 2023-05-24T20:21:58.127Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 790 | 2023-05-24T20:21:58.127Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 791 | 2023-05-24T20:21:58.143Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 792 | 2023-05-24T20:21:58.162Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 793 | 2023-05-24T20:21:58.849Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 223.710017332s
|
| 794 | 2023-05-24T20:21:59.299Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService44"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="82b76c73-bde1-4b97-a719-fffe29da7169"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 795 | 2023-05-24T20:21:59.299Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 796 | 2023-05-24T20:22:00.368Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 225.045449001s
|
| 797 | 2023-05-24T20:22:02.335Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 227.196002868s
|
| 798 | 2023-05-24T20:22:04.012Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService44] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService44 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = de568a43-d251-4b8f-bbf7-677af2d233e6
response_code = 500
uri = /filesystem
|
| 799 | 2023-05-24T20:22:04.015Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "de568a43-d251-4b8f-bbf7-677af2d233e6", "content-length": "124", "date": "Wed, 24 May 2023 20:22:03 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "de568a43-d251-4b8f-bbf7-677af2d233e6" }
|
| 800 | 2023-05-24T20:22:05.030Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 229.707310592s
|
| 801 | 2023-05-24T20:22:05.456Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 802 | 2023-05-24T20:22:05.456Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 803 | 2023-05-24T20:22:05.792Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 230.469442914s
|
| 804 | 2023-05-24T20:22:07.026Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 231.886996156s
|
| 805 | 2023-05-24T20:22:08.516Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 233.376420267s
|
| 806 | 2023-05-24T20:22:09.810Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 807 | 2023-05-24T20:22:09.810Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 808 | 2023-05-24T20:22:09.829Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 809 | 2023-05-24T20:22:09.829Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 810 | 2023-05-24T20:22:09.829Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 811 | 2023-05-24T20:22:09.829Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 812 | 2023-05-24T20:22:09.829Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 813 | 2023-05-24T20:22:09.829Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 814 | 2023-05-24T20:22:09.845Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 815 | 2023-05-24T20:22:09.865Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 816 | 2023-05-24T20:22:11.031Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService45"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="aec8ca92-363f-4af8-9a89-c01229f278d4"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 817 | 2023-05-24T20:22:11.032Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 818 | 2023-05-24T20:22:11.536Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 235.682336334s
|
| 819 | 2023-05-24T20:22:12.257Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 236.933662374s
|
| 820 | 2023-05-24T20:22:15.817Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService45] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService45 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 5d29134e-ae88-4df3-a123-5e3303a8fa1c
response_code = 500
uri = /filesystem
|
| 821 | 2023-05-24T20:22:15.820Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "5d29134e-ae88-4df3-a123-5e3303a8fa1c", "content-length": "124", "date": "Wed, 24 May 2023 20:22:15 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "5d29134e-ae88-4df3-a123-5e3303a8fa1c" }
|
| 822 | 2023-05-24T20:22:16.689Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 241.365939417s
|
| 823 | 2023-05-24T20:22:17.245Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 824 | 2023-05-24T20:22:17.245Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 825 | 2023-05-24T20:22:18.949Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 826 | 2023-05-24T20:22:18.950Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 827 | 2023-05-24T20:22:18.969Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 828 | 2023-05-24T20:22:18.969Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 829 | 2023-05-24T20:22:18.969Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 830 | 2023-05-24T20:22:18.969Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 831 | 2023-05-24T20:22:18.969Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 832 | 2023-05-24T20:22:18.969Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 833 | 2023-05-24T20:22:18.985Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 834 | 2023-05-24T20:22:19.004Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 835 | 2023-05-24T20:22:20.327Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService46"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="c6facf8b-4ed1-464b-aba8-686cd9c142a8"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 836 | 2023-05-24T20:22:20.328Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 837 | 2023-05-24T20:22:20.986Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 245.663249559s
|
| 838 | 2023-05-24T20:22:21.985Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 246.661840525s
|
| 839 | 2023-05-24T20:22:23.006Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 840 | 2023-05-24T20:22:24.479Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 249.339869535s
|
| 841 | 2023-05-24T20:22:24.729Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 249.590404281s
|
| 842 | 2023-05-24T20:22:24.985Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService46] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService46 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 94c52031-6d1d-4c03-90ea-9d169862154f
response_code = 500
uri = /filesystem
|
| 843 | 2023-05-24T20:22:24.988Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "94c52031-6d1d-4c03-90ea-9d169862154f", "content-length": "124", "date": "Wed, 24 May 2023 20:22:24 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "94c52031-6d1d-4c03-90ea-9d169862154f" }
|
| 844 | 2023-05-24T20:22:26.411Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 845 | 2023-05-24T20:22:26.411Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 846 | 2023-05-24T20:22:29.722Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 254.399256787s
|
| 847 | 2023-05-24T20:22:30.994Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 255.854607849s
|
| 848 | 2023-05-24T20:22:32.882Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 849 | 2023-05-24T20:22:32.882Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 850 | 2023-05-24T20:22:32.902Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 851 | 2023-05-24T20:22:32.902Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 852 | 2023-05-24T20:22:32.902Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 853 | 2023-05-24T20:22:32.902Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 854 | 2023-05-24T20:22:32.902Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 855 | 2023-05-24T20:22:32.902Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 856 | 2023-05-24T20:22:32.918Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 857 | 2023-05-24T20:22:32.936Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 858 | 2023-05-24T20:22:33.206Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 257.883177177s
|
| 859 | 2023-05-24T20:22:34.073Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService47"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="4f62d47c-491e-4b15-a52f-c984e3dc1525"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 860 | 2023-05-24T20:22:34.074Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 861 | 2023-05-24T20:22:34.121Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 258.981581544s
|
| 862 | 2023-05-24T20:22:36.482Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 261.15949783s
|
| 863 | 2023-05-24T20:22:37.451Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 262.127976794s
|
| 864 | 2023-05-24T20:22:38.008Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 262.153568713s
|
| 865 | 2023-05-24T20:22:39.363Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService47] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService47 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = ca5218f9-741a-4d8f-8c01-6ffeb1ab3c40
response_code = 500
uri = /filesystem
|
| 866 | 2023-05-24T20:22:39.366Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "ca5218f9-741a-4d8f-8c01-6ffeb1ab3c40", "content-length": "124", "date": "Wed, 24 May 2023 20:22:39 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "ca5218f9-741a-4d8f-8c01-6ffeb1ab3c40" }
|
| 867 | 2023-05-24T20:22:40.805Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 868 | 2023-05-24T20:22:40.805Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 869 | 2023-05-24T20:22:44.727Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }
|
| 870 | 2023-05-24T20:22:44.728Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 871 | 2023-05-24T20:22:44.748Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 872 | 2023-05-24T20:22:44.748Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 873 | 2023-05-24T20:22:44.748Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 874 | 2023-05-24T20:22:44.748Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 875 | 2023-05-24T20:22:44.748Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 876 | 2023-05-24T20:22:44.748Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 877 | 2023-05-24T20:22:44.764Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 878 | 2023-05-24T20:22:44.783Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 879 | 2023-05-24T20:22:44.963Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 269.639790033s
|
| 880 | 2023-05-24T20:22:45.478Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 270.155064607s
|
| 881 | 2023-05-24T20:22:45.948Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService48"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::8"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="875ce6a7-a1aa-45e7-b5b1-c10c6c50fba8"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 882 | 2023-05-24T20:22:45.948Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 883 | 2023-05-24T20:22:50.224Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 43c82571-b757-4b5c-afc0-e8587506dc30
response_code = 204
uri = /filesystem
|
| 884 | 2023-05-24T20:22:50.224Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 885 | 2023-05-24T20:22:50.224Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 886 | 2023-05-24T20:22:50.270Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 887 | 2023-05-24T20:22:50.270Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 888 | 2023-05-24T20:22:50.270Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 889 | 2023-05-24T20:22:50.270Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 890 | 2023-05-24T20:22:50.270Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 891 | 2023-05-24T20:22:50.270Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 892 | 2023-05-24T20:22:50.270Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 893 | 2023-05-24T20:22:50.270Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 894 | 2023-05-24T20:22:50.287Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 895 | 2023-05-24T20:22:50.306Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 896 | 2023-05-24T20:22:50.643Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 897 | 2023-05-24T20:22:51.492Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService49"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::9"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="bccd506b-afd5-491c-a7bf-4a8aed12bdd0"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 898 | 2023-05-24T20:22:51.492Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 899 | 2023-05-24T20:22:52.792Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 277.652747924s
|
| 900 | 2023-05-24T20:22:53.230Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 278.091298869s
|
| 901 | 2023-05-24T20:22:56.621Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Command [/usr/sbin/zlogin oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b /usr/sbin/ipadm create-if -t oxControlService49] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService49 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = d2cf9b0d-f975-432e-9eb2-835d71a30898
response_code = 500
uri = /filesystem
|
| 902 | 2023-05-24T20:22:56.624Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d2cf9b0d-f975-432e-9eb2-835d71a30898", "content-length": "124", "date": "Wed, 24 May 2023 20:22:56 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d2cf9b0d-f975-432e-9eb2-835d71a30898" }
|
| 903 | 2023-05-24T20:22:56.718Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 281.578731814s
|
| 904 | 2023-05-24T20:22:56.744Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 905 | 2023-05-24T20:22:56.744Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 906 | 2023-05-24T20:22:56.763Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 907 | 2023-05-24T20:22:56.763Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 908 | 2023-05-24T20:22:56.763Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 909 | 2023-05-24T20:22:56.763Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 910 | 2023-05-24T20:22:56.763Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 911 | 2023-05-24T20:22:56.763Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 912 | 2023-05-24T20:22:56.763Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 913 | 2023-05-24T20:22:56.763Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 914 | 2023-05-24T20:22:56.779Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown
|
| 915 | 2023-05-24T20:22:56.779Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 916 | 2023-05-24T20:22:56.798Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = b1fbbd3a-ae1f-49f3-812c-cdef7d280e4a
response_code = 500
uri = /filesystem
|
| 917 | 2023-05-24T20:22:56.801Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b1fbbd3a-ae1f-49f3-812c-cdef7d280e4a", "content-length": "124", "date": "Wed, 24 May 2023 20:22:56 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b1fbbd3a-ae1f-49f3-812c-cdef7d280e4a" }
|
| 918 | 2023-05-24T20:22:56.937Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 919 | 2023-05-24T20:22:56.937Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 920 | 2023-05-24T20:22:56.954Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 921 | 2023-05-24T20:22:56.954Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 922 | 2023-05-24T20:22:56.954Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 923 | 2023-05-24T20:22:56.954Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 924 | 2023-05-24T20:22:56.954Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 925 | 2023-05-24T20:22:56.954Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 926 | 2023-05-24T20:22:56.954Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 927 | 2023-05-24T20:22:56.954Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 928 | 2023-05-24T20:22:56.970Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown
|
| 929 | 2023-05-24T20:22:56.970Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 930 | 2023-05-24T20:22:56.989Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = fc02c436-5209-43c0-a496-d0c3364f4be3
response_code = 500
uri = /filesystem
|
| 931 | 2023-05-24T20:22:56.992Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "fc02c436-5209-43c0-a496-d0c3364f4be3", "content-length": "124", "date": "Wed, 24 May 2023 20:22:56 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "fc02c436-5209-43c0-a496-d0c3364f4be3" }
|
| 932 | 2023-05-24T20:22:57.094Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 933 | 2023-05-24T20:22:57.095Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 934 | 2023-05-24T20:22:57.110Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 935 | 2023-05-24T20:22:57.110Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 936 | 2023-05-24T20:22:57.110Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 937 | 2023-05-24T20:22:57.110Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 938 | 2023-05-24T20:22:57.110Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 939 | 2023-05-24T20:22:57.110Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 940 | 2023-05-24T20:22:57.110Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 941 | 2023-05-24T20:22:57.110Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 942 | 2023-05-24T20:22:57.124Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown
|
| 943 | 2023-05-24T20:22:57.124Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 944 | 2023-05-24T20:22:57.144Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 3c38a430-8167-4272-a6fd-0bf71f2118bd
response_code = 500
uri = /filesystem
|
| 945 | 2023-05-24T20:22:57.147Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3c38a430-8167-4272-a6fd-0bf71f2118bd", "content-length": "124", "date": "Wed, 24 May 2023 20:22:56 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3c38a430-8167-4272-a6fd-0bf71f2118bd" }
|
| 946 | 2023-05-24T20:22:57.332Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 947 | 2023-05-24T20:22:57.333Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 948 | 2023-05-24T20:22:57.348Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 949 | 2023-05-24T20:22:57.348Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 950 | 2023-05-24T20:22:57.348Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 951 | 2023-05-24T20:22:57.348Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 952 | 2023-05-24T20:22:57.348Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 953 | 2023-05-24T20:22:57.348Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 954 | 2023-05-24T20:22:57.348Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 955 | 2023-05-24T20:22:57.348Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 956 | 2023-05-24T20:22:57.363Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown
|
| 957 | 2023-05-24T20:22:57.363Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 958 | 2023-05-24T20:22:57.383Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 40a6b44e-ac16-46b6-9514-ac70d672b7dc
response_code = 500
uri = /filesystem
|
| 959 | 2023-05-24T20:22:57.385Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "40a6b44e-ac16-46b6-9514-ac70d672b7dc", "content-length": "124", "date": "Wed, 24 May 2023 20:22:56 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "40a6b44e-ac16-46b6-9514-ac70d672b7dc" }
|
| 960 | 2023-05-24T20:22:57.592Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 961 | 2023-05-24T20:22:57.592Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 962 | 2023-05-24T20:22:57.609Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 963 | 2023-05-24T20:22:57.609Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 964 | 2023-05-24T20:22:57.609Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 965 | 2023-05-24T20:22:57.609Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 966 | 2023-05-24T20:22:57.609Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 967 | 2023-05-24T20:22:57.609Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 968 | 2023-05-24T20:22:57.609Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 969 | 2023-05-24T20:22:57.609Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 970 | 2023-05-24T20:22:57.623Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown
|
| 971 | 2023-05-24T20:22:57.623Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 972 | 2023-05-24T20:22:57.642Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 9c71743c-6037-4d06-8a95-f763d9716d62
response_code = 500
uri = /filesystem
|
| 973 | 2023-05-24T20:22:57.645Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "9c71743c-6037-4d06-8a95-f763d9716d62", "content-length": "124", "date": "Wed, 24 May 2023 20:22:56 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "9c71743c-6037-4d06-8a95-f763d9716d62" }
|
| 974 | 2023-05-24T20:22:57.787Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 975 | 2023-05-24T20:22:57.787Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 976 | 2023-05-24T20:22:57.804Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 977 | 2023-05-24T20:22:57.804Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 978 | 2023-05-24T20:22:57.804Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 979 | 2023-05-24T20:22:57.804Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 980 | 2023-05-24T20:22:57.804Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 981 | 2023-05-24T20:22:57.804Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 982 | 2023-05-24T20:22:57.804Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 983 | 2023-05-24T20:22:57.804Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 984 | 2023-05-24T20:22:57.820Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state Incomplete
|
| 985 | 2023-05-24T20:22:57.820Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 986 | 2023-05-24T20:22:58.044Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 987 | 2023-05-24T20:22:58.044Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 988 | 2023-05-24T20:22:58.044Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 25\nstdout:\nINFO: omicron: uninstalling zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b...\nstderr:\nzoneadm: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b: could not reset state: Problem updating index file
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 70b82277-195a-432e-b7e8-a40ba0fcb222
response_code = 500
uri = /filesystem
|
| 989 | 2023-05-24T20:22:58.051Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "70b82277-195a-432e-b7e8-a40ba0fcb222", "content-length": "124", "date": "Wed, 24 May 2023 20:22:57 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "70b82277-195a-432e-b7e8-a40ba0fcb222" }
|
| 990 | 2023-05-24T20:22:58.357Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 991 | 2023-05-24T20:22:58.358Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 992 | 2023-05-24T20:22:58.377Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 993 | 2023-05-24T20:22:58.377Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 994 | 2023-05-24T20:22:58.377Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 995 | 2023-05-24T20:22:58.377Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 996 | 2023-05-24T20:22:58.377Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 997 | 2023-05-24T20:22:58.377Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 998 | 2023-05-24T20:22:58.377Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 999 | 2023-05-24T20:22:58.377Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1000 | 2023-05-24T20:22:58.393Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1001 | 2023-05-24T20:22:58.413Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1002 | 2023-05-24T20:22:59.622Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService56"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::9"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="0fda899f-dea0-4a5c-80bb-8cb24853aede"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1003 | 2023-05-24T20:22:59.622Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1004 | 2023-05-24T20:23:00.214Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 284.891366935s
|
| 1005 | 2023-05-24T20:23:02.420Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 287.097267796s
|
| 1006 | 2023-05-24T20:23:04.283Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Command [/usr/sbin/zlogin oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b /usr/sbin/ipadm create-if -t oxControlService56] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService56 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 85b58404-56e7-4ec1-b987-a20f4f23a185
response_code = 500
uri = /filesystem
|
| 1007 | 2023-05-24T20:23:04.286Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "85b58404-56e7-4ec1-b987-a20f4f23a185", "content-length": "124", "date": "Wed, 24 May 2023 20:23:04 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "85b58404-56e7-4ec1-b987-a20f4f23a185" }
|
| 1008 | 2023-05-24T20:23:04.571Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 1009 | 2023-05-24T20:23:04.571Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1010 | 2023-05-24T20:23:04.589Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1011 | 2023-05-24T20:23:04.590Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1012 | 2023-05-24T20:23:04.590Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1013 | 2023-05-24T20:23:04.590Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1014 | 2023-05-24T20:23:04.590Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1015 | 2023-05-24T20:23:04.590Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1016 | 2023-05-24T20:23:04.590Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1017 | 2023-05-24T20:23:04.590Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1018 | 2023-05-24T20:23:04.605Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown
|
| 1019 | 2023-05-24T20:23:04.605Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1020 | 2023-05-24T20:23:04.626Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 88561607-5431-4536-9ac1-54945e6e4a1e
response_code = 500
uri = /filesystem
|
| 1021 | 2023-05-24T20:23:04.629Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "88561607-5431-4536-9ac1-54945e6e4a1e", "content-length": "124", "date": "Wed, 24 May 2023 20:23:04 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "88561607-5431-4536-9ac1-54945e6e4a1e" }
|
| 1022 | 2023-05-24T20:23:04.682Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 289.542623353s
|
| 1023 | 2023-05-24T20:23:05.017Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 1024 | 2023-05-24T20:23:05.017Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1025 | 2023-05-24T20:23:05.035Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1026 | 2023-05-24T20:23:05.035Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1027 | 2023-05-24T20:23:05.035Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1028 | 2023-05-24T20:23:05.035Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1029 | 2023-05-24T20:23:05.035Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1030 | 2023-05-24T20:23:05.035Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1031 | 2023-05-24T20:23:05.035Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1032 | 2023-05-24T20:23:05.035Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1033 | 2023-05-24T20:23:05.051Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown
|
| 1034 | 2023-05-24T20:23:05.051Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1035 | 2023-05-24T20:23:05.070Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 21a8048f-522f-4394-bea8-ead8834d64c6
response_code = 500
uri = /filesystem
|
| 1036 | 2023-05-24T20:23:05.072Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "21a8048f-522f-4394-bea8-ead8834d64c6", "content-length": "124", "date": "Wed, 24 May 2023 20:23:04 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "21a8048f-522f-4394-bea8-ead8834d64c6" }
|
| 1037 | 2023-05-24T20:23:05.475Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 1038 | 2023-05-24T20:23:05.476Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1039 | 2023-05-24T20:23:05.495Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1040 | 2023-05-24T20:23:05.495Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1041 | 2023-05-24T20:23:05.495Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1042 | 2023-05-24T20:23:05.495Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1043 | 2023-05-24T20:23:05.495Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1044 | 2023-05-24T20:23:05.495Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1045 | 2023-05-24T20:23:05.495Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1046 | 2023-05-24T20:23:05.495Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1047 | 2023-05-24T20:23:05.513Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state Incomplete
|
| 1048 | 2023-05-24T20:23:05.513Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1049 | 2023-05-24T20:23:05.645Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 289.791197965s
|
| 1050 | 2023-05-24T20:23:05.719Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1051 | 2023-05-24T20:23:05.719Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1052 | 2023-05-24T20:23:05.724Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b: No such zone configured\noxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b: No such zone configured
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 123b7ac6-4fd0-457b-8836-2fa260933726
response_code = 500
uri = /filesystem
|
| 1053 | 2023-05-24T20:23:05.732Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "123b7ac6-4fd0-457b-8836-2fa260933726", "content-length": "124", "date": "Wed, 24 May 2023 20:23:05 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "123b7ac6-4fd0-457b-8836-2fa260933726" }
|
| 1054 | 2023-05-24T20:23:06.180Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 1055 | 2023-05-24T20:23:06.180Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1056 | 2023-05-24T20:23:06.198Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1057 | 2023-05-24T20:23:06.198Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1058 | 2023-05-24T20:23:06.198Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1059 | 2023-05-24T20:23:06.198Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1060 | 2023-05-24T20:23:06.198Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1061 | 2023-05-24T20:23:06.198Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1062 | 2023-05-24T20:23:06.198Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1063 | 2023-05-24T20:23:06.198Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1064 | 2023-05-24T20:23:06.214Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1065 | 2023-05-24T20:23:06.232Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1066 | 2023-05-24T20:23:07.361Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService60"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::9"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="a05545d5-c397-4978-ac2a-74e28230daf2"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1067 | 2023-05-24T20:23:07.361Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1068 | 2023-05-24T20:23:08.802Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 293.479198439s
|
| 1069 | 2023-05-24T20:23:11.804Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 296.481146383s
|
| 1070 | 2023-05-24T20:23:12.096Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Command [/usr/sbin/zlogin oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b /usr/sbin/ipadm create-if -t oxControlService60] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService60 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 88b73d9e-21b9-4a10-9780-b5f09193d9d8
response_code = 500
uri = /filesystem
|
| 1071 | 2023-05-24T20:23:12.099Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "88b73d9e-21b9-4a10-9780-b5f09193d9d8", "content-length": "124", "date": "Wed, 24 May 2023 20:23:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "88b73d9e-21b9-4a10-9780-b5f09193d9d8" }
|
| 1072 | 2023-05-24T20:23:12.488Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 1073 | 2023-05-24T20:23:12.488Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1074 | 2023-05-24T20:23:12.504Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1075 | 2023-05-24T20:23:12.504Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1076 | 2023-05-24T20:23:12.504Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1077 | 2023-05-24T20:23:12.504Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1078 | 2023-05-24T20:23:12.504Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1079 | 2023-05-24T20:23:12.504Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1080 | 2023-05-24T20:23:12.504Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1081 | 2023-05-24T20:23:12.504Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1082 | 2023-05-24T20:23:12.518Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown
|
| 1083 | 2023-05-24T20:23:12.518Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1084 | 2023-05-24T20:23:12.538Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 8a79b514-94ca-4b0f-8272-048450a3eab4
response_code = 500
uri = /filesystem
|
| 1085 | 2023-05-24T20:23:12.540Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8a79b514-94ca-4b0f-8272-048450a3eab4", "content-length": "124", "date": "Wed, 24 May 2023 20:23:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8a79b514-94ca-4b0f-8272-048450a3eab4" }
|
| 1086 | 2023-05-24T20:23:13.530Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1087 | 2023-05-24T20:23:13.531Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1088 | 2023-05-24T20:23:13.814Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 1089 | 2023-05-24T20:23:13.814Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1090 | 2023-05-24T20:23:13.833Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1091 | 2023-05-24T20:23:13.833Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1092 | 2023-05-24T20:23:13.833Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1093 | 2023-05-24T20:23:13.833Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1094 | 2023-05-24T20:23:13.833Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1095 | 2023-05-24T20:23:13.833Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1096 | 2023-05-24T20:23:13.833Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1097 | 2023-05-24T20:23:13.833Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1098 | 2023-05-24T20:23:13.849Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1099 | 2023-05-24T20:23:13.869Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1100 | 2023-05-24T20:23:15.003Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService62"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::9"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="8b10b517-ddb0-45af-a85f-6f1c1bc24250"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1101 | 2023-05-24T20:23:15.003Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1102 | 2023-05-24T20:23:16.399Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 301.076112431s
|
| 1103 | 2023-05-24T20:23:17.761Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 302.621594194s
|
| 1104 | 2023-05-24T20:23:19.973Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Command [/usr/sbin/zlogin oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b /usr/sbin/ipadm create-if -t oxControlService62] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService62 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 56d28c14-8717-4294-a173-1e7b178ac360
response_code = 500
uri = /filesystem
|
| 1105 | 2023-05-24T20:23:19.976Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "56d28c14-8717-4294-a173-1e7b178ac360", "content-length": "124", "date": "Wed, 24 May 2023 20:23:19 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "56d28c14-8717-4294-a173-1e7b178ac360" }
|
| 1106 | 2023-05-24T20:23:20.039Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 304.716591563s
|
| 1107 | 2023-05-24T20:23:21.052Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 305.912565391s
|
| 1108 | 2023-05-24T20:23:21.114Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 1109 | 2023-05-24T20:23:21.114Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1110 | 2023-05-24T20:23:21.136Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1111 | 2023-05-24T20:23:21.136Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1112 | 2023-05-24T20:23:21.136Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1113 | 2023-05-24T20:23:21.136Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1114 | 2023-05-24T20:23:21.136Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1115 | 2023-05-24T20:23:21.136Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1116 | 2023-05-24T20:23:21.136Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1117 | 2023-05-24T20:23:21.136Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1118 | 2023-05-24T20:23:21.154Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state Installed
|
| 1119 | 2023-05-24T20:23:21.154Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1120 | 2023-05-24T20:23:21.415Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1121 | 2023-05-24T20:23:21.415Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1122 | 2023-05-24T20:23:21.420Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b: No such zone configured\noxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b: No such zone configured
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = b1d3b5ce-42e6-41d6-9017-25fe2f930412
response_code = 500
uri = /filesystem
|
| 1123 | 2023-05-24T20:23:21.427Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b1d3b5ce-42e6-41d6-9017-25fe2f930412", "content-length": "124", "date": "Wed, 24 May 2023 20:23:21 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b1d3b5ce-42e6-41d6-9017-25fe2f930412" }
|
| 1124 | 2023-05-24T20:23:21.548Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 1125 | 2023-05-24T20:23:22.502Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }
|
| 1126 | 2023-05-24T20:23:22.502Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1127 | 2023-05-24T20:23:22.521Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1128 | 2023-05-24T20:23:22.521Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1129 | 2023-05-24T20:23:22.521Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1130 | 2023-05-24T20:23:22.521Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1131 | 2023-05-24T20:23:22.521Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1132 | 2023-05-24T20:23:22.521Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1133 | 2023-05-24T20:23:22.521Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1134 | 2023-05-24T20:23:22.521Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1135 | 2023-05-24T20:23:22.537Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1136 | 2023-05-24T20:23:22.557Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1137 | 2023-05-24T20:23:23.677Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService64"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::9"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="786f2305-2eaf-4c0f-8f12-041ceb6cd74e"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1138 | 2023-05-24T20:23:23.677Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b
|
| 1139 | 2023-05-24T20:23:26.500Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 311.361239609s
|
| 1140 | 2023-05-24T20:23:28.188Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = f45c6050-3c0f-4c31-84b4-4fdb954b2127
response_code = 204
uri = /filesystem
|
| 1141 | 2023-05-24T20:23:28.188Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1142 | 2023-05-24T20:23:28.188Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1143 | 2023-05-24T20:23:28.229Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1144 | 2023-05-24T20:23:28.229Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1145 | 2023-05-24T20:23:28.229Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1146 | 2023-05-24T20:23:28.229Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1147 | 2023-05-24T20:23:28.229Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1148 | 2023-05-24T20:23:28.229Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1149 | 2023-05-24T20:23:28.229Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1150 | 2023-05-24T20:23:28.229Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1151 | 2023-05-24T20:23:28.229Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1152 | 2023-05-24T20:23:28.229Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1153 | 2023-05-24T20:23:28.245Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1154 | 2023-05-24T20:23:28.264Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1155 | 2023-05-24T20:23:29.392Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService65"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::a"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="6d9390c9-4989-486b-ae29-ea475f66f5b7"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1156 | 2023-05-24T20:23:29.392Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1157 | 2023-05-24T20:23:30.255Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 315.115393507s
|
| 1158 | 2023-05-24T20:23:34.395Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService65] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService65 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = f67b19c2-d8e7-4684-afc7-7caf6c8aaedc
response_code = 500
uri = /filesystem
|
| 1159 | 2023-05-24T20:23:34.398Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f67b19c2-d8e7-4684-afc7-7caf6c8aaedc", "content-length": "124", "date": "Wed, 24 May 2023 20:23:34 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f67b19c2-d8e7-4684-afc7-7caf6c8aaedc" }
|
| 1160 | 2023-05-24T20:23:34.484Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1161 | 2023-05-24T20:23:34.485Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1162 | 2023-05-24T20:23:34.503Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1163 | 2023-05-24T20:23:34.503Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1164 | 2023-05-24T20:23:34.503Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1165 | 2023-05-24T20:23:34.503Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1166 | 2023-05-24T20:23:34.503Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1167 | 2023-05-24T20:23:34.503Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1168 | 2023-05-24T20:23:34.503Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1169 | 2023-05-24T20:23:34.503Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1170 | 2023-05-24T20:23:34.503Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1171 | 2023-05-24T20:23:34.503Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1172 | 2023-05-24T20:23:34.519Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 1173 | 2023-05-24T20:23:34.519Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1174 | 2023-05-24T20:23:34.538Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = f80e7c1c-b51e-472f-b446-737bdc15f1f6
response_code = 500
uri = /filesystem
|
| 1175 | 2023-05-24T20:23:34.541Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "f80e7c1c-b51e-472f-b446-737bdc15f1f6", "content-length": "124", "date": "Wed, 24 May 2023 20:23:34 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "f80e7c1c-b51e-472f-b446-737bdc15f1f6" }
|
| 1176 | 2023-05-24T20:23:34.680Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1177 | 2023-05-24T20:23:34.681Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1178 | 2023-05-24T20:23:34.699Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1179 | 2023-05-24T20:23:34.699Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1180 | 2023-05-24T20:23:34.699Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1181 | 2023-05-24T20:23:34.699Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1182 | 2023-05-24T20:23:34.699Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1183 | 2023-05-24T20:23:34.699Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1184 | 2023-05-24T20:23:34.699Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1185 | 2023-05-24T20:23:34.699Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1186 | 2023-05-24T20:23:34.699Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1187 | 2023-05-24T20:23:34.699Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1188 | 2023-05-24T20:23:34.715Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 1189 | 2023-05-24T20:23:34.715Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1190 | 2023-05-24T20:23:34.734Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = b2f1254d-616a-4e62-883d-bba989ac818f
response_code = 500
uri = /filesystem
|
| 1191 | 2023-05-24T20:23:34.736Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b2f1254d-616a-4e62-883d-bba989ac818f", "content-length": "124", "date": "Wed, 24 May 2023 20:23:34 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b2f1254d-616a-4e62-883d-bba989ac818f" }
|
| 1192 | 2023-05-24T20:23:34.938Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1193 | 2023-05-24T20:23:34.938Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1194 | 2023-05-24T20:23:34.954Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1195 | 2023-05-24T20:23:34.955Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1196 | 2023-05-24T20:23:34.955Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1197 | 2023-05-24T20:23:34.955Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1198 | 2023-05-24T20:23:34.955Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1199 | 2023-05-24T20:23:34.955Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1200 | 2023-05-24T20:23:34.955Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1201 | 2023-05-24T20:23:34.955Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1202 | 2023-05-24T20:23:34.955Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1203 | 2023-05-24T20:23:34.955Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1204 | 2023-05-24T20:23:34.971Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 1205 | 2023-05-24T20:23:34.971Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1206 | 2023-05-24T20:23:34.990Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = fb9dce53-88c2-452f-b0d6-1cf1d4b005f9
response_code = 500
uri = /filesystem
|
| 1207 | 2023-05-24T20:23:34.993Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "fb9dce53-88c2-452f-b0d6-1cf1d4b005f9", "content-length": "124", "date": "Wed, 24 May 2023 20:23:34 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "fb9dce53-88c2-452f-b0d6-1cf1d4b005f9" }
|
| 1208 | 2023-05-24T20:23:35.233Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1209 | 2023-05-24T20:23:35.233Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1210 | 2023-05-24T20:23:35.251Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1211 | 2023-05-24T20:23:35.251Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1212 | 2023-05-24T20:23:35.251Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1213 | 2023-05-24T20:23:35.251Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1214 | 2023-05-24T20:23:35.251Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1215 | 2023-05-24T20:23:35.251Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1216 | 2023-05-24T20:23:35.251Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1217 | 2023-05-24T20:23:35.251Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1218 | 2023-05-24T20:23:35.251Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1219 | 2023-05-24T20:23:35.251Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1220 | 2023-05-24T20:23:35.267Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 1221 | 2023-05-24T20:23:35.267Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1222 | 2023-05-24T20:23:35.286Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = fc073abf-62a4-41ba-90a4-000e8ef8d6ed
response_code = 500
uri = /filesystem
|
| 1223 | 2023-05-24T20:23:35.289Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "fc073abf-62a4-41ba-90a4-000e8ef8d6ed", "content-length": "124", "date": "Wed, 24 May 2023 20:23:34 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "fc073abf-62a4-41ba-90a4-000e8ef8d6ed" }
|
| 1224 | 2023-05-24T20:23:35.554Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1225 | 2023-05-24T20:23:35.554Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1226 | 2023-05-24T20:23:35.574Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1227 | 2023-05-24T20:23:35.574Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1228 | 2023-05-24T20:23:35.574Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1229 | 2023-05-24T20:23:35.574Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1230 | 2023-05-24T20:23:35.574Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1231 | 2023-05-24T20:23:35.574Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1232 | 2023-05-24T20:23:35.574Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1233 | 2023-05-24T20:23:35.574Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1234 | 2023-05-24T20:23:35.574Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1235 | 2023-05-24T20:23:35.574Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1236 | 2023-05-24T20:23:35.592Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Incomplete
|
| 1237 | 2023-05-24T20:23:35.592Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1238 | 2023-05-24T20:23:35.824Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1239 | 2023-05-24T20:23:35.824Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1240 | 2023-05-24T20:23:35.829Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured\noxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = e6d7805f-8a19-449f-9166-b40861b370c8
response_code = 500
uri = /filesystem
|
| 1241 | 2023-05-24T20:23:35.836Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "e6d7805f-8a19-449f-9166-b40861b370c8", "content-length": "124", "date": "Wed, 24 May 2023 20:23:35 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "e6d7805f-8a19-449f-9166-b40861b370c8" }
|
| 1242 | 2023-05-24T20:23:35.965Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1243 | 2023-05-24T20:23:35.965Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1244 | 2023-05-24T20:23:35.983Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1245 | 2023-05-24T20:23:35.983Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1246 | 2023-05-24T20:23:35.983Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1247 | 2023-05-24T20:23:35.983Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1248 | 2023-05-24T20:23:35.983Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1249 | 2023-05-24T20:23:35.983Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1250 | 2023-05-24T20:23:35.983Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1251 | 2023-05-24T20:23:35.983Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1252 | 2023-05-24T20:23:35.983Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1253 | 2023-05-24T20:23:35.983Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1254 | 2023-05-24T20:23:35.994Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 320.671195473s
|
| 1255 | 2023-05-24T20:23:35.998Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1256 | 2023-05-24T20:23:36.015Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1257 | 2023-05-24T20:23:36.550Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 320.696419483s
|
| 1258 | 2023-05-24T20:23:37.137Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 321.814591113s
|
| 1259 | 2023-05-24T20:23:37.253Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService71"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::a"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="4c01b083-6c63-45d8-b2ae-2c3bfa1bf6a7"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1260 | 2023-05-24T20:23:37.253Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1261 | 2023-05-24T20:23:41.462Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 326.13908484s
|
| 1262 | 2023-05-24T20:23:41.734Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService71] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService71 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 05639170-c7cf-40e6-8ee6-bc26bd4f27bb
response_code = 500
uri = /filesystem
|
| 1263 | 2023-05-24T20:23:41.738Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "05639170-c7cf-40e6-8ee6-bc26bd4f27bb", "content-length": "124", "date": "Wed, 24 May 2023 20:23:41 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "05639170-c7cf-40e6-8ee6-bc26bd4f27bb" }
|
| 1264 | 2023-05-24T20:23:42.082Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1265 | 2023-05-24T20:23:42.083Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1266 | 2023-05-24T20:23:42.101Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1267 | 2023-05-24T20:23:42.101Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1268 | 2023-05-24T20:23:42.101Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1269 | 2023-05-24T20:23:42.101Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1270 | 2023-05-24T20:23:42.101Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1271 | 2023-05-24T20:23:42.101Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1272 | 2023-05-24T20:23:42.101Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1273 | 2023-05-24T20:23:42.101Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1274 | 2023-05-24T20:23:42.101Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1275 | 2023-05-24T20:23:42.101Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1276 | 2023-05-24T20:23:42.119Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 1277 | 2023-05-24T20:23:42.119Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1278 | 2023-05-24T20:23:42.139Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = aba87740-e493-402d-b329-43ed705bdb0c
response_code = 500
uri = /filesystem
|
| 1279 | 2023-05-24T20:23:42.142Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "aba87740-e493-402d-b329-43ed705bdb0c", "content-length": "124", "date": "Wed, 24 May 2023 20:23:42 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "aba87740-e493-402d-b329-43ed705bdb0c" }
|
| 1280 | 2023-05-24T20:23:42.533Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1281 | 2023-05-24T20:23:42.533Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1282 | 2023-05-24T20:23:42.551Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1283 | 2023-05-24T20:23:42.551Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1284 | 2023-05-24T20:23:42.551Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1285 | 2023-05-24T20:23:42.551Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1286 | 2023-05-24T20:23:42.551Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1287 | 2023-05-24T20:23:42.551Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1288 | 2023-05-24T20:23:42.551Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1289 | 2023-05-24T20:23:42.551Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1290 | 2023-05-24T20:23:42.551Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1291 | 2023-05-24T20:23:42.551Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1292 | 2023-05-24T20:23:42.567Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown
|
| 1293 | 2023-05-24T20:23:42.567Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1294 | 2023-05-24T20:23:42.587Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 5f84dfdd-267e-4e0a-bff9-ed7a10c7822d
response_code = 500
uri = /filesystem
|
| 1295 | 2023-05-24T20:23:42.589Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "5f84dfdd-267e-4e0a-bff9-ed7a10c7822d", "content-length": "124", "date": "Wed, 24 May 2023 20:23:41 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "5f84dfdd-267e-4e0a-bff9-ed7a10c7822d" }
|
| 1296 | 2023-05-24T20:23:43.020Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1297 | 2023-05-24T20:23:43.021Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1298 | 2023-05-24T20:23:43.040Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1299 | 2023-05-24T20:23:43.040Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1300 | 2023-05-24T20:23:43.040Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1301 | 2023-05-24T20:23:43.040Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1302 | 2023-05-24T20:23:43.040Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1303 | 2023-05-24T20:23:43.040Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1304 | 2023-05-24T20:23:43.040Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1305 | 2023-05-24T20:23:43.040Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1306 | 2023-05-24T20:23:43.040Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1307 | 2023-05-24T20:23:43.040Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1308 | 2023-05-24T20:23:43.057Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Incomplete
|
| 1309 | 2023-05-24T20:23:43.057Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1310 | 2023-05-24T20:23:43.170Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1311 | 2023-05-24T20:23:43.170Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1312 | 2023-05-24T20:23:43.174Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured\noxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = 8b3d6b73-939e-4e99-b085-eb49e196505d
response_code = 500
uri = /filesystem
|
| 1313 | 2023-05-24T20:23:43.182Z | WARN | SledAgent (RSS): failed to create filesystem
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "8b3d6b73-939e-4e99-b085-eb49e196505d", "content-length": "124", "date": "Wed, 24 May 2023 20:23:43 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "8b3d6b73-939e-4e99-b085-eb49e196505d" }
|
| 1314 | 2023-05-24T20:23:43.825Z | INFO | SledAgent (RSS): creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }
|
| 1315 | 2023-05-24T20:23:43.825Z | INFO | SledAgent (StorageManager): add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }
|
| 1316 | 2023-05-24T20:23:43.844Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CockroachDb
|
| 1317 | 2023-05-24T20:23:43.844Z | INFO | SledAgent (ServiceManager): Service zone cockroachdb already exists
|
| 1318 | 2023-05-24T20:23:43.844Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Clickhouse
|
| 1319 | 2023-05-24T20:23:43.844Z | INFO | SledAgent (ServiceManager): Service zone clickhouse already exists
|
| 1320 | 2023-05-24T20:23:43.844Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1321 | 2023-05-24T20:23:43.844Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1322 | 2023-05-24T20:23:43.844Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1323 | 2023-05-24T20:23:43.844Z | INFO | SledAgent (ServiceManager): Service zone crucible already exists
|
| 1324 | 2023-05-24T20:23:43.844Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Crucible
|
| 1325 | 2023-05-24T20:23:43.844Z | INFO | SledAgent (ServiceManager): Service zone crucible does not yet exist
|
| 1326 | 2023-05-24T20:23:43.861Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1327 | 2023-05-24T20:23:43.879Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1328 | 2023-05-24T20:23:44.671Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 329.53175481s
|
| 1329 | 2023-05-24T20:23:44.993Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/agent">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService75"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="dataset" value="oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::a"/>
<propval type="astring" name="listen_port" value="32345"/>
<propval type="astring" name="uuid" value="6b6bd813-268b-49e7-9f0c-9aea40b12649"/>
<propval type="astring" name="store" value="/data"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1330 | 2023-05-24T20:23:44.993Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03
|
| 1331 | 2023-05-24T20:23:47.877Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 332.554240648s
|
| 1332 | 2023-05-24T20:23:48.870Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 333.547703036s
|
| 1333 | 2023-05-24T20:23:49.305Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:50743
req_id = cad2ecb8-fecc-4583-9a1a-3ef5265eb872
response_code = 204
uri = /filesystem
|
| 1334 | 2023-05-24T20:23:49.306Z | INFO | SledAgent (RSS): Finished setting up agents and datasets
|
| 1335 | 2023-05-24T20:23:49.306Z | INFO | SledAgent (RSS): sending service requests...
|
| 1336 | 2023-05-24T20:23:49.306Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1337 | 2023-05-24T20:23:49.306Z | INFO | SledAgent (dropshot (SledAgent)): accepted connection
local_addr = [fd00:1122:3344:101::1]:12345
remote_addr = [fd00:1122:3344:101::1]:51451
|
| 1338 | 2023-05-24T20:23:49.307Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1339 | 2023-05-24T20:23:49.307Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1340 | 2023-05-24T20:23:49.326Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry
|
| 1341 | 2023-05-24T20:23:49.343Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry
|
| 1342 | 2023-05-24T20:23:50.256Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService76"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::b"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1343 | 2023-05-24T20:23:50.256Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_pantry
|
| 1344 | 2023-05-24T20:23:50.359Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 335.036208563s
|
| 1345 | 2023-05-24T20:23:54.970Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService76] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService76 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 3d30412f-9f66-43c1-b7f1-b9126592e253
response_code = 500
uri = /services
|
| 1346 | 2023-05-24T20:23:54.974Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "3d30412f-9f66-43c1-b7f1-b9126592e253", "content-length": "124", "date": "Wed, 24 May 2023 20:23:54 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "3d30412f-9f66-43c1-b7f1-b9126592e253" }
|
| 1347 | 2023-05-24T20:23:55.118Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1348 | 2023-05-24T20:23:55.118Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Oximeter
|
| 1349 | 2023-05-24T20:23:55.118Z | INFO | SledAgent (ServiceManager): Service zone oximeter does not yet exist
|
| 1350 | 2023-05-24T20:23:55.136Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_oximeter
|
| 1351 | 2023-05-24T20:23:55.154Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_oximeter
|
| 1352 | 2023-05-24T20:23:55.807Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 340.667362661s
|
| 1353 | 2023-05-24T20:23:56.108Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_oximeter
|
| 1354 | 2023-05-24T20:23:56.620Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_pantry
|
| 1355 | 2023-05-24T20:23:56.620Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_pantry
|
| 1356 | 2023-05-24T20:24:00.721Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::5 exists
|
| 1357 | 2023-05-24T20:24:00.721Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::5, prefix: 64 }))
zone = oxz_oximeter
|
| 1358 | 2023-05-24T20:24:01.259Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::5 exists - OK
|
| 1359 | 2023-05-24T20:24:01.259Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
| 1360 | 2023-05-24T20:24:01.409Z | INFO | SledAgent (ServiceManager): Setting up oximeter service
|
| 1361 | 2023-05-24T20:24:01.517Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1362 | 2023-05-24T20:24:01.517Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1363 | 2023-05-24T20:24:01.539Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry
|
| 1364 | 2023-05-24T20:24:01.558Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry
|
| 1365 | 2023-05-24T20:24:02.368Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 347.22904362s
|
| 1366 | 2023-05-24T20:24:02.456Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService78"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::b"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1367 | 2023-05-24T20:24:02.456Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_pantry
|
| 1368 | 2023-05-24T20:24:04.054Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 348.915040811s
|
| 1369 | 2023-05-24T20:24:04.106Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 1370 | 2023-05-24T20:24:06.384Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 351.060789659s
|
| 1371 | 2023-05-24T20:24:06.997Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService78] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService78 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = c9eac7c2-7a9c-4d2d-afc8-cbe5e3115c1e
response_code = 500
uri = /services
|
| 1372 | 2023-05-24T20:24:07.000Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "c9eac7c2-7a9c-4d2d-afc8-cbe5e3115c1e", "content-length": "124", "date": "Wed, 24 May 2023 20:24:06 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "c9eac7c2-7a9c-4d2d-afc8-cbe5e3115c1e" }
|
| 1373 | 2023-05-24T20:24:07.163Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1374 | 2023-05-24T20:24:07.164Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1375 | 2023-05-24T20:24:07.164Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1376 | 2023-05-24T20:24:07.181Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown
|
| 1377 | 2023-05-24T20:24:07.181Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_pantry
|
| 1378 | 2023-05-24T20:24:07.201Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 6b1513b8-795b-472a-b039-58509d8ea63a
response_code = 500
uri = /services
|
| 1379 | 2023-05-24T20:24:07.204Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6b1513b8-795b-472a-b039-58509d8ea63a", "content-length": "124", "date": "Wed, 24 May 2023 20:24:07 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6b1513b8-795b-472a-b039-58509d8ea63a" }
|
| 1380 | 2023-05-24T20:24:07.375Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1381 | 2023-05-24T20:24:07.376Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Oximeter
|
| 1382 | 2023-05-24T20:24:07.376Z | INFO | SledAgent (ServiceManager): Service zone oximeter already exists
|
| 1383 | 2023-05-24T20:24:07.376Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: ExternalDns
|
| 1384 | 2023-05-24T20:24:07.376Z | INFO | SledAgent (ServiceManager): Service zone external_dns does not yet exist
|
| 1385 | 2023-05-24T20:24:07.384Z | INFO | SledAgent (PortManager): Created OPTE port
port = Port { inner: PortInner { name: "opte1", _ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 255, 212, 24]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: "vopte1" } }
|
| 1386 | 2023-05-24T20:24:07.384Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
service = ServiceZoneService { id: 8885de6b-6073-47f8-921c-c148b35c79d3, details: ExternalDns { http_address: [fd00:1122:3344:101::3]:5353, dns_address: 192.168.1.20:53, nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, kind: Service { id: 8885de6b-6073-47f8-921c-c148b35c79d3 }, name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 24])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }
|
| 1387 | 2023-05-24T20:24:07.402Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_external_dns
|
| 1388 | 2023-05-24T20:24:07.420Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_external_dns
|
| 1389 | 2023-05-24T20:24:08.580Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_pantry
|
| 1390 | 2023-05-24T20:24:08.580Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_pantry
|
| 1391 | 2023-05-24T20:24:08.657Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_external_dns
|
| 1392 | 2023-05-24T20:24:12.334Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 357.011275079s
|
| 1393 | 2023-05-24T20:24:13.255Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::3 exists
|
| 1394 | 2023-05-24T20:24:13.255Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::3, prefix: 64 }))
zone = oxz_external_dns
|
| 1395 | 2023-05-24T20:24:13.793Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::3 exists - OK
|
| 1396 | 2023-05-24T20:24:13.793Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
| 1397 | 2023-05-24T20:24:13.943Z | INFO | SledAgent (ServiceManager): Setting up external-dns service
|
| 1398 | 2023-05-24T20:24:13.943Z | INFO | SledAgent (ServiceManager): Ensuring address for OPTE port
zone = oxz_external_dns
|
| 1399 | 2023-05-24T20:24:20.494Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Nexus
|
| 1400 | 2023-05-24T20:24:20.494Z | INFO | SledAgent (ServiceManager): Service zone nexus does not yet exist
|
| 1401 | 2023-05-24T20:24:20.495Z | WARN | SledAgent: failed to notify nexus about sled agent
error = Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
total duration = 364.640937704s
|
| 1402 | 2023-05-24T20:24:20.495Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 365.356140139s
|
| 1403 | 2023-05-24T20:24:20.495Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 365.172618771s
|
| 1404 | 2023-05-24T20:24:20.507Z | INFO | SledAgent (PortManager): Created OPTE port
port = Port { inner: PortInner { name: "opte2", _ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 178, 86]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: "vopte2" } }
|
| 1405 | 2023-05-24T20:24:20.507Z | INFO | SledAgent (ServiceManager): creating NAT entry for service
service = ServiceZoneService { id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, details: Nexus { internal_ip: fd00:1122:3344:101::4, external_ip: 192.168.1.21, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, kind: Service { id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }, name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }
|
| 1406 | 2023-05-24T20:24:20.527Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_nexus
|
| 1407 | 2023-05-24T20:24:20.546Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_nexus
|
| 1408 | 2023-05-24T20:24:21.947Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_nexus
|
| 1409 | 2023-05-24T20:24:23.130Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 367.807460257s
|
| 1410 | 2023-05-24T20:24:24.990Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 369.850928146s
|
| 1411 | 2023-05-24T20:24:27.043Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::4 exists
|
| 1412 | 2023-05-24T20:24:27.043Z | INFO | SledAgent (ServiceManager): Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::4, prefix: 64 }))
zone = oxz_nexus
|
| 1413 | 2023-05-24T20:24:27.583Z | INFO | SledAgent (ServiceManager): Ensuring address fd00:1122:3344:101::4 exists - OK
|
| 1414 | 2023-05-24T20:24:27.583Z | INFO | SledAgent (ServiceManager): GZ addresses: []
|
| 1415 | 2023-05-24T20:24:27.733Z | INFO | SledAgent (ServiceManager): Setting up Nexus service
|
| 1416 | 2023-05-24T20:24:27.733Z | INFO | SledAgent (ServiceManager): Ensuring address for OPTE port
zone = oxz_nexus
|
| 1417 | 2023-05-24T20:24:34.177Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 379.038096629s
|
| 1418 | 2023-05-24T20:24:34.178Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }
total duration = 379.039142468s
|
| 1419 | 2023-05-24T20:24:34.179Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: U2 }
total duration = 378.856082471s
|
| 1420 | 2023-05-24T20:24:34.179Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 378.856194575s
|
| 1421 | 2023-05-24T20:24:34.179Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }, variant: U2 }
total duration = 378.85639882s
|
| 1422 | 2023-05-24T20:24:34.180Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 378.85716464s
|
| 1423 | 2023-05-24T20:24:34.203Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1424 | 2023-05-24T20:24:34.203Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1425 | 2023-05-24T20:24:34.216Z | WARN | SledAgent (StorageManager): failed to notify nexus about Add { identity: DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b", model: "synthetic-model" }, variant: M2 }
total duration = 379.076640139s
|
| 1426 | 2023-05-24T20:24:34.221Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry
|
| 1427 | 2023-05-24T20:24:34.240Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry
|
| 1428 | 2023-05-24T20:24:35.374Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService82"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::b"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1429 | 2023-05-24T20:24:35.374Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_pantry
|
| 1430 | 2023-05-24T20:24:40.670Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 385.346777768s
|
| 1431 | 2023-05-24T20:24:40.766Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService82] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService82 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = b7b07d2c-3acf-4819-8239-5885ed879215
response_code = 500
uri = /services
|
| 1432 | 2023-05-24T20:24:40.769Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "b7b07d2c-3acf-4819-8239-5885ed879215", "content-length": "124", "date": "Wed, 24 May 2023 20:24:40 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "b7b07d2c-3acf-4819-8239-5885ed879215" }
|
| 1433 | 2023-05-24T20:24:41.010Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1434 | 2023-05-24T20:24:41.011Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1435 | 2023-05-24T20:24:41.011Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1436 | 2023-05-24T20:24:41.029Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown
|
| 1437 | 2023-05-24T20:24:41.029Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_pantry
|
| 1438 | 2023-05-24T20:24:41.051Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 97a970c1-27bd-4c1f-b5a3-3ba2716629b1
response_code = 500
uri = /services
|
| 1439 | 2023-05-24T20:24:41.054Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "97a970c1-27bd-4c1f-b5a3-3ba2716629b1", "content-length": "124", "date": "Wed, 24 May 2023 20:24:41 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "97a970c1-27bd-4c1f-b5a3-3ba2716629b1" }
|
| 1440 | 2023-05-24T20:24:41.324Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1441 | 2023-05-24T20:24:41.325Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Nexus
|
| 1442 | 2023-05-24T20:24:41.325Z | INFO | SledAgent (ServiceManager): Service zone nexus already exists
|
| 1443 | 2023-05-24T20:24:41.325Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1444 | 2023-05-24T20:24:41.325Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1445 | 2023-05-24T20:24:41.342Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown
|
| 1446 | 2023-05-24T20:24:41.342Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_pantry
|
| 1447 | 2023-05-24T20:24:41.364Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 2a269637-c724-4cf7-a458-34fe9a41509c
response_code = 500
uri = /services
|
| 1448 | 2023-05-24T20:24:41.367Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "2a269637-c724-4cf7-a458-34fe9a41509c", "content-length": "124", "date": "Wed, 24 May 2023 20:24:41 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "2a269637-c724-4cf7-a458-34fe9a41509c" }
|
| 1449 | 2023-05-24T20:24:41.678Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1450 | 2023-05-24T20:24:41.679Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Oximeter
|
| 1451 | 2023-05-24T20:24:41.679Z | INFO | SledAgent (ServiceManager): Service zone oximeter already exists
|
| 1452 | 2023-05-24T20:24:41.679Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Nexus
|
| 1453 | 2023-05-24T20:24:41.679Z | INFO | SledAgent (ServiceManager): Service zone nexus already exists
|
| 1454 | 2023-05-24T20:24:41.679Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: ExternalDns
|
| 1455 | 2023-05-24T20:24:41.679Z | INFO | SledAgent (ServiceManager): Service zone external_dns already exists
|
| 1456 | 2023-05-24T20:24:41.679Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1457 | 2023-05-24T20:24:41.679Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1458 | 2023-05-24T20:24:41.696Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown
|
| 1459 | 2023-05-24T20:24:41.696Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_pantry
|
| 1460 | 2023-05-24T20:24:41.718Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = dcf66fbc-a745-4f7c-a041-c6c695fefc6f
response_code = 500
uri = /services
|
| 1461 | 2023-05-24T20:24:41.721Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "dcf66fbc-a745-4f7c-a041-c6c695fefc6f", "content-length": "124", "date": "Wed, 24 May 2023 20:24:41 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "dcf66fbc-a745-4f7c-a041-c6c695fefc6f" }
|
| 1462 | 2023-05-24T20:24:41.968Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1463 | 2023-05-24T20:24:41.968Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1464 | 2023-05-24T20:24:41.968Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1465 | 2023-05-24T20:24:41.987Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_pantry in state Incomplete
|
| 1466 | 2023-05-24T20:24:41.987Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_pantry
|
| 1467 | 2023-05-24T20:24:42.181Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_pantry
|
| 1468 | 2023-05-24T20:24:42.181Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_pantry
|
| 1469 | 2023-05-24T20:24:42.182Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 25\nstdout:\nINFO: omicron: uninstalling zone oxz_crucible_pantry...\nstderr:\nzoneadm: oxz_crucible_pantry: could not reset state: Problem updating index file
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = d6a73482-81b1-4252-b856-c8457a1cc77a
response_code = 500
uri = /services
|
| 1470 | 2023-05-24T20:24:42.185Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "d6a73482-81b1-4252-b856-c8457a1cc77a", "content-length": "124", "date": "Wed, 24 May 2023 20:24:42 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "d6a73482-81b1-4252-b856-c8457a1cc77a" }
|
| 1471 | 2023-05-24T20:24:42.679Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1472 | 2023-05-24T20:24:42.679Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Nexus
|
| 1473 | 2023-05-24T20:24:42.679Z | INFO | SledAgent (ServiceManager): Service zone nexus already exists
|
| 1474 | 2023-05-24T20:24:42.679Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Oximeter
|
| 1475 | 2023-05-24T20:24:42.679Z | INFO | SledAgent (ServiceManager): Service zone oximeter already exists
|
| 1476 | 2023-05-24T20:24:42.679Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1477 | 2023-05-24T20:24:42.679Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1478 | 2023-05-24T20:24:42.698Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry
|
| 1479 | 2023-05-24T20:24:42.717Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry
|
| 1480 | 2023-05-24T20:24:43.796Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService87"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::b"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1481 | 2023-05-24T20:24:43.796Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_pantry
|
| 1482 | 2023-05-24T20:24:47.221Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 1483 | 2023-05-24T20:24:47.227Z | INFO | SledAgent: contacting server nexus, registering sled
baseboard = Baseboard { part_number: "i86pc", revision: 0, serial_number: "da005c867f5e4d1daa7f756de235d90e" }
id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
sled_id = 6f0f865b-7d3c-40ef-a8b9-18c66c253d85
|
| 1484 | 2023-05-24T20:24:48.423Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService87] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService87 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 1b310639-e15e-4a1a-852d-502a0e2db746
response_code = 500
uri = /services
|
| 1485 | 2023-05-24T20:24:48.426Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1b310639-e15e-4a1a-852d-502a0e2db746", "content-length": "124", "date": "Wed, 24 May 2023 20:24:48 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1b310639-e15e-4a1a-852d-502a0e2db746" }
|
| 1486 | 2023-05-24T20:24:49.014Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1487 | 2023-05-24T20:24:49.015Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1488 | 2023-05-24T20:24:49.015Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1489 | 2023-05-24T20:24:49.033Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown
|
| 1490 | 2023-05-24T20:24:49.033Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_pantry
|
| 1491 | 2023-05-24T20:24:49.056Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 510ffea7-c12b-4f4b-b791-7c4a706d6744
response_code = 500
uri = /services
|
| 1492 | 2023-05-24T20:24:49.059Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "510ffea7-c12b-4f4b-b791-7c4a706d6744", "content-length": "124", "date": "Wed, 24 May 2023 20:24:49 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "510ffea7-c12b-4f4b-b791-7c4a706d6744" }
|
| 1493 | 2023-05-24T20:24:49.361Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1494 | 2023-05-24T20:24:49.362Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1495 | 2023-05-24T20:24:49.362Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1496 | 2023-05-24T20:24:49.379Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown
|
| 1497 | 2023-05-24T20:24:49.379Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_pantry
|
| 1498 | 2023-05-24T20:24:49.401Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = cd9fd7e2-ab79-4bbe-aa03-0cd4ab1c8d73
response_code = 500
uri = /services
|
| 1499 | 2023-05-24T20:24:49.404Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "cd9fd7e2-ab79-4bbe-aa03-0cd4ab1c8d73", "content-length": "124", "date": "Wed, 24 May 2023 20:24:48 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "cd9fd7e2-ab79-4bbe-aa03-0cd4ab1c8d73" }
|
| 1500 | 2023-05-24T20:24:49.714Z | WARN | SledAgent (StorageManager): failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: "synthetic-vendor", serial: "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03", model: "synthetic-model" }
total duration = 394.391170581s
|
| 1501 | 2023-05-24T20:24:49.759Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1502 | 2023-05-24T20:24:49.760Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Nexus
|
| 1503 | 2023-05-24T20:24:49.760Z | INFO | SledAgent (ServiceManager): Service zone nexus already exists
|
| 1504 | 2023-05-24T20:24:49.760Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: ExternalDns
|
| 1505 | 2023-05-24T20:24:49.760Z | INFO | SledAgent (ServiceManager): Service zone external_dns already exists
|
| 1506 | 2023-05-24T20:24:49.760Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Oximeter
|
| 1507 | 2023-05-24T20:24:49.760Z | INFO | SledAgent (ServiceManager): Service zone oximeter already exists
|
| 1508 | 2023-05-24T20:24:49.760Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1509 | 2023-05-24T20:24:49.760Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1510 | 2023-05-24T20:24:49.779Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_pantry in state Incomplete
|
| 1511 | 2023-05-24T20:24:49.779Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_pantry
|
| 1512 | 2023-05-24T20:24:49.847Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_pantry
|
| 1513 | 2023-05-24T20:24:49.847Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_pantry
|
| 1514 | 2023-05-24T20:24:49.848Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 25\nstdout:\nINFO: omicron: uninstalling zone oxz_crucible_pantry...\nstderr:\nzoneadm: oxz_crucible_pantry: could not reset state: Problem updating index file
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = ab7d9d5e-8a8e-4db2-922d-a95f4063d3ac
response_code = 500
uri = /services
|
| 1515 | 2023-05-24T20:24:49.848Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "ab7d9d5e-8a8e-4db2-922d-a95f4063d3ac", "content-length": "124", "date": "Wed, 24 May 2023 20:24:49 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "ab7d9d5e-8a8e-4db2-922d-a95f4063d3ac" }
|
| 1516 | 2023-05-24T20:24:50.827Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1517 | 2023-05-24T20:24:50.828Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Oximeter
|
| 1518 | 2023-05-24T20:24:50.828Z | INFO | SledAgent (ServiceManager): Service zone oximeter already exists
|
| 1519 | 2023-05-24T20:24:50.828Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: ExternalDns
|
| 1520 | 2023-05-24T20:24:50.828Z | INFO | SledAgent (ServiceManager): Service zone external_dns already exists
|
| 1521 | 2023-05-24T20:24:50.828Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Nexus
|
| 1522 | 2023-05-24T20:24:50.828Z | INFO | SledAgent (ServiceManager): Service zone nexus already exists
|
| 1523 | 2023-05-24T20:24:50.828Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1524 | 2023-05-24T20:24:50.828Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1525 | 2023-05-24T20:24:50.848Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry
|
| 1526 | 2023-05-24T20:24:50.867Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry
|
| 1527 | 2023-05-24T20:24:51.895Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService91"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::b"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1528 | 2023-05-24T20:24:51.895Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_pantry
|
| 1529 | 2023-05-24T20:24:56.392Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService91] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService91 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 59f14e12-86cb-4474-b5fa-5c2e8e882786
response_code = 500
uri = /services
|
| 1530 | 2023-05-24T20:24:56.396Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "59f14e12-86cb-4474-b5fa-5c2e8e882786", "content-length": "124", "date": "Wed, 24 May 2023 20:24:56 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "59f14e12-86cb-4474-b5fa-5c2e8e882786" }
|
| 1531 | 2023-05-24T20:24:57.263Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1532 | 2023-05-24T20:24:57.264Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Nexus
|
| 1533 | 2023-05-24T20:24:57.264Z | INFO | SledAgent (ServiceManager): Service zone nexus already exists
|
| 1534 | 2023-05-24T20:24:57.264Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1535 | 2023-05-24T20:24:57.264Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1536 | 2023-05-24T20:24:57.283Z | INFO | SledAgent (ServiceManager): install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown
|
| 1537 | 2023-05-24T20:24:57.284Z | INFO | SledAgent (ServiceManager): Invalid state; uninstalling and deleting zone oxz_crucible_pantry
|
| 1538 | 2023-05-24T20:24:57.306Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 7b337b27-75fa-401d-85a8-dbdaace45e22
response_code = 500
uri = /services
|
| 1539 | 2023-05-24T20:24:57.309Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "7b337b27-75fa-401d-85a8-dbdaace45e22", "content-length": "124", "date": "Wed, 24 May 2023 20:24:57 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "7b337b27-75fa-401d-85a8-dbdaace45e22" }
|
| 1540 | 2023-05-24T20:24:57.814Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_pantry
|
| 1541 | 2023-05-24T20:24:57.814Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_pantry
|
| 1542 | 2023-05-24T20:24:58.703Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1543 | 2023-05-24T20:24:58.704Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Nexus
|
| 1544 | 2023-05-24T20:24:58.704Z | INFO | SledAgent (ServiceManager): Service zone nexus already exists
|
| 1545 | 2023-05-24T20:24:58.704Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Oximeter
|
| 1546 | 2023-05-24T20:24:58.704Z | INFO | SledAgent (ServiceManager): Service zone oximeter already exists
|
| 1547 | 2023-05-24T20:24:58.704Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1548 | 2023-05-24T20:24:58.704Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1549 | 2023-05-24T20:24:58.722Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry
|
| 1550 | 2023-05-24T20:24:58.740Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry
|
| 1551 | 2023-05-24T20:24:59.787Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService93"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::b"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1552 | 2023-05-24T20:24:59.787Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_pantry
|
| 1553 | 2023-05-24T20:25:04.628Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService93] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService93 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 1e19cc4e-64cd-4c65-a238-debab66824a5
response_code = 500
uri = /services
|
| 1554 | 2023-05-24T20:25:04.632Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "1e19cc4e-64cd-4c65-a238-debab66824a5", "content-length": "124", "date": "Wed, 24 May 2023 20:25:04 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "1e19cc4e-64cd-4c65-a238-debab66824a5" }
|
| 1555 | 2023-05-24T20:25:06.057Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_pantry
|
| 1556 | 2023-05-24T20:25:06.057Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_pantry
|
| 1557 | 2023-05-24T20:25:06.519Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1558 | 2023-05-24T20:25:06.520Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1559 | 2023-05-24T20:25:06.520Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1560 | 2023-05-24T20:25:06.539Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry
|
| 1561 | 2023-05-24T20:25:06.558Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry
|
| 1562 | 2023-05-24T20:25:07.605Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService94"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::b"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1563 | 2023-05-24T20:25:07.605Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_pantry
|
| 1564 | 2023-05-24T20:25:12.323Z | INFO | SledAgent (dropshot (SledAgent)): request completed
error_message_external = Internal Server Error
error_message_internal = Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService94] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService94 : Interface already exists
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 212bf972-0d60-44aa-aad0-8c8c0d4c4e5c
response_code = 500
uri = /services
|
| 1565 | 2023-05-24T20:25:12.329Z | WARN | SledAgent (RSS): failed to initialize services
error = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "212bf972-0d60-44aa-aad0-8c8c0d4c4e5c", "content-length": "124", "date": "Wed, 24 May 2023 20:25:12 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "212bf972-0d60-44aa-aad0-8c8c0d4c4e5c" }
|
| 1566 | 2023-05-24T20:25:13.763Z | INFO | SledAgent (ServiceManager): halt_and_remove_logged: Previous zone state: Running
zone = oxz_crucible_pantry
|
| 1567 | 2023-05-24T20:25:13.763Z | INFO | SledAgent (ServiceManager): Stopped and uninstalled zone
zone = oxz_crucible_pantry
|
| 1568 | 2023-05-24T20:25:14.477Z | INFO | SledAgent (RSS): initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: "192.168.1.20:53", http_address: "[fd00:1122:3344:101::3]:5353", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr("A8:40:25:FF:D4:18"), name: Name("external-dns-8885de6b-6073-47f8-921c-c148b35c79d3"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.0.0/24")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr("A8:40:25:FF:B2:56"), name: Name("nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.1.0/24")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: "[fd00:1122:3344:1::1]:53", http_address: "[fd00:1122:3344:1::1]:5353" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: ["1.1.1.1", "9.9.9.9"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr("A8:40:25:FF:D5:51"), name: Name("ntp-0d607e9a-f782-4173-b84a-04929f7be674"), primary: true, slot: 0, subnet: V4(Ipv4Net("172.30.2.0/24")), vni: Vni(100) }, ntp_servers: ["ntp.eng.oxide.computer"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]
|
| 1569 | 2023-05-24T20:25:14.478Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: CruciblePantry
|
| 1570 | 2023-05-24T20:25:14.478Z | INFO | SledAgent (ServiceManager): Service zone crucible_pantry does not yet exist
|
| 1571 | 2023-05-24T20:25:14.497Z | INFO | SledAgent (ServiceManager): Configuring new Omicron zone: oxz_crucible_pantry
|
| 1572 | 2023-05-24T20:25:14.516Z | INFO | SledAgent (ServiceManager): Installing Omicron zone: oxz_crucible_pantry
|
| 1573 | 2023-05-24T20:25:15.575Z | INFO | SledAgent (ServiceManager): Profile for oxz_crucible_pantry:
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="omicron">
<service version="1" type="service" name="oxide/crucible/pantry">
<instance enabled="true" name="default">
<property_group type="application" name="config">
<propval type="astring" name="datalink" value="oxControlService95"/>
<propval type="astring" name="gateway" value="fd00:1122:3344:101::1"/>
<propval type="astring" name="listen_addr" value="fd00:1122:3344:101::b"/>
<propval type="astring" name="listen_port" value="17000"/>
</property_group>
</instance>
</service>
</service_bundle>
|
| 1574 | 2023-05-24T20:25:15.575Z | INFO | SledAgent (ServiceManager): Zone booting
zone = oxz_crucible_pantry
|
| 1575 | 2023-05-24T20:25:19.854Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Nexus
|
| 1576 | 2023-05-24T20:25:19.854Z | INFO | SledAgent (ServiceManager): Service zone nexus already exists
|
| 1577 | 2023-05-24T20:25:19.854Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: ExternalDns
|
| 1578 | 2023-05-24T20:25:19.854Z | INFO | SledAgent (ServiceManager): Service zone external_dns already exists
|
| 1579 | 2023-05-24T20:25:19.854Z | INFO | SledAgent (ServiceManager): Ensuring service zone is initialized: Oximeter
|
| 1580 | 2023-05-24T20:25:19.854Z | INFO | SledAgent (ServiceManager): Service zone oximeter already exists
|
| 1581 | 2023-05-24T20:25:19.855Z | INFO | SledAgent (dropshot (SledAgent)): request completed
local_addr = [fd00:1122:3344:101::1]:12345
method = PUT
remote_addr = [fd00:1122:3344:101::1]:51451
req_id = 2f24cb02-efae-44fc-969b-34093ea0b823
response_code = 204
uri = /services
|
| 1582 | 2023-05-24T20:25:19.855Z | INFO | SledAgent (RSS): Finished setting up services
|
| 1583 | 2023-05-24T20:25:19.856Z | INFO | SledAgent (RSS): Handing off control to Nexus
|
| 1584 | 2023-05-24T20:25:19.856Z | INFO | SledAgent (DnsResolver): new DNS resolver
addresses = [[fd00:1122:3344:1::1]:53]
|
| 1585 | 2023-05-24T20:25:19.857Z | INFO | SledAgent (RSS): Nexus address: [fd00:1122:3344:101::4]:12221
|
| 1586 | 2023-05-24T20:25:20.430Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "f5705cf0-3aed-4b20-8521-77aba3c42d56", "content-length": "174", "date": "Wed, 24 May 2023 20:25:19 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"f4b4dc87-ab46-49fb-a4b4-d361ae214c03\"", request_id: "f5705cf0-3aed-4b20-8521-77aba3c42d56" }
|
| 1587 | 2023-05-24T20:25:20.615Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "e780198e-cdba-4bf2-b0ce-727824bf20d3", "content-length": "174", "date": "Wed, 24 May 2023 20:25:19 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"f4b4dc87-ab46-49fb-a4b4-d361ae214c03\"", request_id: "e780198e-cdba-4bf2-b0ce-727824bf20d3" }
|
| 1588 | 2023-05-24T20:25:20.836Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "9203c3c2-7556-4c72-b666-3404fb1c7668", "content-length": "174", "date": "Wed, 24 May 2023 20:25:19 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"f4b4dc87-ab46-49fb-a4b4-d361ae214c03\"", request_id: "9203c3c2-7556-4c72-b666-3404fb1c7668" }
|
| 1589 | 2023-05-24T20:25:21.061Z | INFO | SledAgent (RSS): Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {"content-type": "application/json", "x-request-id": "bb4f93b5-0f8a-405a-8acc-d53fed0a1f01", "content-length": "174", "date": "Wed, 24 May 2023 20:25:20 GMT"}; value: Error { error_code: Some("ObjectNotFound"), message: "not found: zpool with id \"f4b4dc87-ab46-49fb-a4b4-d361ae214c03\"", request_id: "bb4f93b5-0f8a-405a-8acc-d53fed0a1f01" }
|