[ May 24 20:18:13 Enabled. ]
[ May 24 20:18:13 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ May 24 20:18:13 Method "start" exited with status 0. ]
[ May 24 20:18:13 Rereading configuration. ]
[ May 24 20:18:13 No 'refresh' method defined. Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.042297313Z","hostname":"sock","pid":1097}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.042518519Z","hostname":"sock","pid":1097,"path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.160948462Z","hostname":"sock","pid":1097,"interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.16941842Z","hostname":"sock","pid":1097}
{"msg":"detecting (real or simulated) SP","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.177002505Z","hostname":"sock","pid":1097}
{"msg":"setting up bootstrap agent server","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.177027876Z","hostname":"sock","pid":1097}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.686876454Z","hostname":"sock","pid":1097,"DdmAdminClient":"[::1]:8000","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"}
{"msg":"Bootstrap Agent monitoring for hardware","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.821694129Z","hostname":"sock","pid":1097}
{"msg":"Starting hardware monitor","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.859596041Z","hostname":"sock","pid":1097,"component":"BootstrapAgent","sled_mode":"Scrimlet { asic: SoftNpu }"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.859632152Z","hostname":"sock","pid":1097,"component":"HardwareManager","component":"BootstrapAgent"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"393b90b4de3e437ead4e4684e3451fb2\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.862507569Z","hostname":"sock","pid":1097,"component":"HardwareManager","component":"BootstrapAgent"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:14.86251668Z","hostname":"sock","pid":1097,"component":"HardwareManager","component":"BootstrapAgent"}
{"msg":"Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.862551871Z","hostname":"sock","pid":1097,"component":"BootstrapAgent"}
{"msg":"Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.862556961Z","hostname":"sock","pid":1097,"component":"BootstrapAgent"}
{"msg":"Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.862563281Z","hostname":"sock","pid":1097,"component":"BootstrapAgent"}
{"msg":"Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.862569551Z","hostname":"sock","pid":1097,"component":"BootstrapAgent"}
{"msg":"Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.862573351Z","hostname":"sock","pid":1097,"component":"BootstrapAgent"}
{"msg":"Upserting synthetic disk for: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.862690074Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.867197966Z","hostname":"sock","pid":1097,"component":"HardwareManager","component":"BootstrapAgent"}
{"msg":"Performing full hardware scan","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.867225317Z","hostname":"sock","pid":1097,"component":"BootstrapAgent"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.867229097Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.867287398Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Storage manager processing zpool: ZpoolInfo {\n name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n size: 10200547328,\n allocated: 402432,\n free: 10200144896,\n health: Online,\n}","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.995456334Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Upserting synthetic disk for: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.995524566Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Found boot disk M.2: (DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal })","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:14.995640719Z","hostname":"sock","pid":1097,"component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.009369839Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.027260111Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Storage manager processing zpool: ZpoolInfo {\n name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n size: 10200547328,\n allocated: 396288,\n free: 10200151040,\n health: Online,\n}","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.139458537Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.139627451Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.139689363Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.139709223Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.139715194Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Upserting synthetic disk for: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.139727644Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.142307713Z","hostname":"sock","pid":1097,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.142384686Z","hostname":"sock","pid":1097,"component":"sprockets (BootstrapAgent)","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n rack_subnet: fd00:1122:3344:100::,\n bootstrap_discovery: OnlyOurs,\n rack_secret_threshold: 1,\n ntp_servers: [\n \"ntp.eng.oxide.computer\",\n ],\n dns_servers: [\n \"1.1.1.1\",\n \"9.9.9.9\",\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 192.168.1.20,\n last: 192.168.1.22,\n },\n ),\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: [\n Certificate {\n cert: [\n 45,\n 45,\n 45,\n 45,\n 45,\n 66,\n 69,\n 71,\n 73,\n 78,\n 32,\n 67,\n 69,\n 82,\n 84,\n 73,\n 70,\n 73,\n 67,\n 65,\n 84,\n 69,\n 45,\n 45,\n 45,\n 45,\n 45,\n 10,\n 77,\n 73,\n 73,\n 70,\n 66,\n 106,\n 67,\n 67,\n 65,\n 117,\n 54,\n 103,\n 65,\n 119,\n 73,\n 66,\n 65,\n 103,\n 73,\n 85,\n 85,\n 77,\n 99,\n 47,\n 81,\n 72,\n 76,\n 83,\n 55,\n 48,\n 103,\n 69,\n 99,\n 75,\n 76,\n 97,\n 112,\n 108,\n 67,\n 79,\n 98,\n 87,\n 101,\n 104,\n 88,\n 74,\n 73,\n 119,\n 68,\n 81,\n 89,\n 74,\n 75,\n 111,\n 90,\n 73,\n 104,\n 118,\n 99,\n 78,\n 65,\n 81,\n 69,\n 76,\n 10,\n 66,\n 81,\n 65,\n 119,\n 73,\n 106,\n 69,\n 103,\n 77,\n 66,\n 52,\n 71,\n 65,\n 49,\n 85,\n 69,\n 65,\n 119,\n 119,\n 88,\n 99,\n 109,\n 86,\n 106,\n 98,\n 51,\n 90,\n 108,\n 99,\n 110,\n 107,\n 117,\n 99,\n 51,\n 108,\n 122,\n 76,\n 109,\n 57,\n 52,\n 97,\n 87,\n 82,\n 108,\n 76,\n 110,\n 82,\n 108,\n 99,\n 51,\n 81,\n 119,\n 72,\n 104,\n 99,\n 78,\n 77,\n 106,\n 77,\n 119,\n 78,\n 84,\n 73,\n 48,\n 10,\n 77,\n 106,\n 65,\n 120,\n 79,\n 68,\n 69,\n 122,\n 87,\n 104,\n 99,\n 78,\n 77,\n 106,\n 77,\n 119,\n 78,\n 84,\n 73,\n 51,\n 77,\n 106,\n 65,\n 120,\n 79,\n 68,\n 69,\n 122,\n 87,\n 106,\n 65,\n 105,\n 77,\n 83,\n 65,\n 119,\n 72,\n 103,\n 89,\n 68,\n 86,\n 81,\n 81,\n 68,\n 68,\n 66,\n 100,\n 121,\n 90,\n 87,\n 78,\n 118,\n 100,\n 109,\n 86,\n 121,\n 101,\n 83,\n 53,\n 122,\n 101,\n 88,\n 77,\n 117,\n 10,\n 98,\n 51,\n 104,\n 112,\n 90,\n 71,\n 85,\n 117,\n 100,\n 71,\n 86,\n 122,\n 100,\n 68,\n 67,\n 67,\n 65,\n 105,\n 73,\n 119,\n 68,\n 81,\n 89,\n 74,\n 75,\n 111,\n 90,\n 73,\n 104,\n 118,\n 99,\n 78,\n 65,\n 81,\n 69,\n 66,\n 66,\n 81,\n 65,\n 68,\n 103,\n 103,\n 73,\n 80,\n 65,\n 68,\n 67,\n 67,\n 65,\n 103,\n 111,\n 67,\n 103,\n 103,\n 73,\n 66,\n 65,\n 77,\n 73,\n 116,\n 82,\n 81,\n 104,\n 74,\n 10,\n 101,\n 67,\n 77,\n 89,\n 76,\n 85,\n 108,\n 88,\n 43,\n 89,\n 57,\n 109,\n 78,\n 74,\n 117,\n 47,\n 119,\n 115,\n 83,\n 100,\n 111,\n 49,\n 114,\n 106,\n 88,\n 68,\n 51,\n 70,\n 53,\n 47,\n 79,\n 43,\n 76,\n 67,\n 73,\n 109,\n 85,\n 57,\n 89,\n 121,\n 75,\n 115,\n 81,\n 77,\n 104,\n 54,\n 56,\n 75,\n 70,\n 83,\n 72,\n 54,\n 121,\n 69,\n 76,\n 99,\n 122,\n 120,\n 88,\n 72,\n 108,\n 119,\n 101,\n 50,\n 10,\n 53,\n 52,\n 50,\n 110,\n 103,\n 107,\n 112,\n 80,\n 84,\n 72,\n 70,\n 90,\n 50,\n 47,\n 50,\n 55,\n 74,\n 90,\n 89,\n 101,\n 81,\n 109,\n 111,\n 110,\n 51,\n 79,\n 75,\n 43,\n 107,\n 98,\n 122,\n 79,\n 55,\n 102,\n 79,\n 83,\n 107,\n 49,\n 54,\n 113,\n 112,\n 73,\n 55,\n 79,\n 121,\n 72,\n 72,\n 66,\n 122,\n 112,\n 88,\n 48,\n 52,\n 110,\n 51,\n 53,\n 74,\n 83,\n 83,\n 113,\n 52,\n 114,\n 97,\n 68,\n 10,\n 80,\n 114,\n 114,\n 81,\n 83,\n 55,\n 109,\n 52,\n 109,\n 87,\n 112,\n 112,\n 113,\n 87,\n 87,\n 107,\n 69,\n 68,\n 49,\n 52,\n 47,\n 49,\n 75,\n 114,\n 109,\n 116,\n 47,\n 87,\n 102,\n 55,\n 88,\n 99,\n 109,\n 53,\n 57,\n 54,\n 85,\n 117,\n 79,\n 53,\n 77,\n 65,\n 50,\n 105,\n 47,\n 48,\n 55,\n 118,\n 51,\n 85,\n 104,\n 100,\n 81,\n 119,\n 84,\n 80,\n 52,\n 110,\n 82,\n 78,\n 76,\n 103,\n 120,\n 52,\n 10,\n 66,\n 106,\n 121,\n 80,\n 101,\n 110,\n 90,\n 71,\n 78,\n 82,\n 120,\n 78,\n 71,\n 106,\n 72,\n 80,\n 67,\n 120,\n 50,\n 116,\n 106,\n 107,\n 71,\n 69,\n 80,\n 80,\n 83,\n 65,\n 76,\n 72,\n 109,\n 48,\n 52,\n 65,\n 69,\n 116,\n 74,\n 99,\n 97,\n 49,\n 81,\n 48,\n 117,\n 65,\n 79,\n 76,\n 86,\n 109,\n 54,\n 80,\n 108,\n 65,\n 115,\n 110,\n 101,\n 82,\n 115,\n 85,\n 97,\n 107,\n 70,\n 50,\n 97,\n 72,\n 10,\n 69,\n 110,\n 66,\n 115,\n 70,\n 118,\n 107,\n 113,\n 99,\n 43,\n 121,\n 90,\n 109,\n 111,\n 86,\n 83,\n 109,\n 107,\n 102,\n 79,\n 89,\n 111,\n 65,\n 117,\n 106,\n 73,\n 78,\n 85,\n 114,\n 69,\n 88,\n 106,\n 87,\n 73,\n 66,\n 54,\n 121,\n 53,\n 107,\n 111,\n 49,\n 119,\n 76,\n 66,\n 73,\n 88,\n 67,\n 85,\n 101,\n 70,\n 51,\n 113,\n 90,\n 102,\n 88,\n 47,\n 118,\n 80,\n 81,\n 47,\n 89,\n 49,\n 74,\n 118,\n 10,\n 54,\n 107,\n 116,\n 121,\n 102,\n 87,\n 90,\n 103,\n 67,\n 84,\n 75,\n 116,\n 55,\n 49,\n 109,\n 82,\n 99,\n 49,\n 47,\n 73,\n 50,\n 53,\n 119,\n 103,\n 51,\n 52,\n 122,\n 55,\n 104,\n 114,\n 105,\n 68,\n 81,\n 98,\n 76,\n 115,\n 114,\n 66,\n 69,\n 116,\n 43,\n 80,\n 66,\n 81,\n 85,\n 52,\n 70,\n 100,\n 122,\n 88,\n 89,\n 55,\n 72,\n 83,\n 55,\n 69,\n 88,\n 81,\n 108,\n 105,\n 117,\n 55,\n 97,\n 54,\n 10,\n 66,\n 49,\n 78,\n 108,\n 106,\n 89,\n 110,\n 103,\n 49,\n 100,\n 110,\n 74,\n 106,\n 110,\n 52,\n 116,\n 79,\n 119,\n 54,\n 104,\n 104,\n 110,\n 43,\n 122,\n 109,\n 116,\n 57,\n 53,\n 121,\n 105,\n 118,\n 89,\n 116,\n 53,\n 67,\n 97,\n 70,\n 117,\n 110,\n 49,\n 90,\n 65,\n 70,\n 72,\n 69,\n 82,\n 85,\n 68,\n 110,\n 119,\n 121,\n 53,\n 106,\n 75,\n 66,\n 48,\n 47,\n 81,\n 55,\n 80,\n 102,\n 98,\n 106,\n 65,\n 10,\n 53,\n 81,\n 76,\n 106,\n 122,\n 51,\n 69,\n 119,\n 84,\n 111,\n 108,\n 73,\n 57,\n 67,\n 110,\n 109,\n 102,\n 105,\n 48,\n 76,\n 50,\n 55,\n 105,\n 113,\n 90,\n 85,\n 107,\n 116,\n 100,\n 55,\n 80,\n 104,\n 119,\n 105,\n 84,\n 53,\n 72,\n 73,\n 55,\n 71,\n 43,\n 47,\n 86,\n 113,\n 106,\n 48,\n 55,\n 74,\n 107,\n 83,\n 85,\n 75,\n 47,\n 114,\n 51,\n 50,\n 74,\n 105,\n 43,\n 84,\n 111,\n 66,\n 51,\n 99,\n 10,\n 74,\n 105,\n 76,\n 74,\n 72,\n 103,\n 48,\n 106,\n 122,\n 103,\n 98,\n 80,\n 78,\n 65,\n 118,\n 98,\n 116,\n 68,\n 49,\n 57,\n 49,\n 100,\n 108,\n 83,\n 107,\n 116,\n 74,\n 120,\n 116,\n 86,\n 120,\n 101,\n 110,\n 55,\n 50,\n 50,\n 66,\n 69,\n 83,\n 71,\n 69,\n 81,\n 104,\n 56,\n 88,\n 97,\n 117,\n 107,\n 77,\n 102,\n 119,\n 105,\n 69,\n 97,\n 50,\n 67,\n 102,\n 86,\n 118,\n 120,\n 68,\n 97,\n 114,\n 47,\n 10,\n 66,\n 116,\n 101,\n 109,\n 110,\n 77,\n 84,\n 116,\n 122,\n 52,\n 53,\n 84,\n 75,\n 113,\n 83,\n 72,\n 49,\n 112,\n 80,\n 50,\n 101,\n 107,\n 107,\n 84,\n 78,\n 84,\n 120,\n 118,\n 52,\n 79,\n 76,\n 106,\n 54,\n 55,\n 66,\n 85,\n 65,\n 86,\n 50,\n 84,\n 120,\n 77,\n 115,\n 66,\n 98,\n 76,\n 113,\n 100,\n 103,\n 53,\n 89,\n 108,\n 113,\n 47,\n 118,\n 70,\n 105,\n 65,\n 110,\n 79,\n 112,\n 100,\n 82,\n 119,\n 10,\n 106,\n 90,\n 71,\n 88,\n 86,\n 72,\n 75,\n 54,\n 111,\n 53,\n 65,\n 57,\n 87,\n 105,\n 47,\n 88,\n 54,\n 86,\n 114,\n 89,\n 107,\n 118,\n 107,\n 47,\n 81,\n 71,\n 53,\n 71,\n 52,\n 102,\n 68,\n 71,\n 99,\n 56,\n 112,\n 84,\n 65,\n 103,\n 77,\n 66,\n 65,\n 65,\n 71,\n 106,\n 78,\n 68,\n 65,\n 121,\n 77,\n 67,\n 73,\n 71,\n 65,\n 49,\n 85,\n 100,\n 69,\n 81,\n 81,\n 98,\n 77,\n 66,\n 109,\n 67,\n 10,\n 70,\n 51,\n 74,\n 108,\n 89,\n 50,\n 57,\n 50,\n 90,\n 88,\n 74,\n 53,\n 76,\n 110,\n 78,\n 53,\n 99,\n 121,\n 53,\n 118,\n 101,\n 71,\n 108,\n 107,\n 90,\n 83,\n 53,\n 48,\n 90,\n 88,\n 78,\n 48,\n 77,\n 65,\n 119,\n 71,\n 65,\n 49,\n 85,\n 100,\n 69,\n 119,\n 69,\n 66,\n 47,\n 119,\n 81,\n 67,\n 77,\n 65,\n 65,\n 119,\n 68,\n 81,\n 89,\n 74,\n 75,\n 111,\n 90,\n 73,\n 104,\n 118,\n 99,\n 78,\n 10,\n 65,\n 81,\n 69,\n 76,\n 66,\n 81,\n 65,\n 68,\n 103,\n 103,\n 73,\n 66,\n 65,\n 71,\n 97,\n 74,\n 66,\n 105,\n 90,\n 68,\n 78,\n 89,\n 115,\n 90,\n 67,\n 67,\n 87,\n 47,\n 67,\n 102,\n 85,\n 89,\n 65,\n 53,\n 105,\n 43,\n 111,\n 84,\n 74,\n 101,\n 122,\n 111,\n 115,\n 78,\n 105,\n 70,\n 112,\n 113,\n 116,\n 48,\n 83,\n 51,\n 100,\n 86,\n 85,\n 87,\n 107,\n 100,\n 121,\n 76,\n 68,\n 112,\n 109,\n 81,\n 10,\n 73,\n 87,\n 97,\n 118,\n 100,\n 112,\n 99,\n 97,\n 80,\n 113,\n 88,\n 73,\n 102,\n 81,\n 66,\n 68,\n 103,\n 74,\n 74,\n 72,\n 69,\n 51,\n 54,\n 66,\n 78,\n 55,\n 43,\n 72,\n 53,\n 72,\n 48,\n 88,\n 108,\n 122,\n 113,\n 102,\n 111,\n 103,\n 66,\n 113,\n 121,\n 68,\n 82,\n 101,\n 109,\n 72,\n 77,\n 101,\n 84,\n 120,\n 113,\n 102,\n 76,\n 107,\n 69,\n 53,\n 71,\n 69,\n 69,\n 115,\n 72,\n 104,\n 49,\n 52,\n 10,\n 104,\n 56,\n 73,\n 74,\n 57,\n 104,\n 105,\n 117,\n 121,\n 90,\n 57,\n 80,\n 52,\n 83,\n 105,\n 72,\n 109,\n 121,\n 74,\n 104,\n 105,\n 118,\n 119,\n 115,\n 51,\n 90,\n 121,\n 107,\n 114,\n 73,\n 55,\n 100,\n 122,\n 53,\n 53,\n 114,\n 102,\n 55,\n 54,\n 84,\n 72,\n 107,\n 89,\n 79,\n 105,\n 121,\n 74,\n 80,\n 72,\n 82,\n 113,\n 55,\n 52,\n 102,\n 97,\n 50,\n 97,\n 81,\n 80,\n 117,\n 85,\n 100,\n 120,\n 56,\n 10,\n 110,\n 102,\n 67,\n 79,\n 71,\n 83,\n 88,\n 109,\n 86,\n 97,\n 77,\n 71,\n 99,\n 111,\n 103,\n 75,\n 119,\n 118,\n 104,\n 53,\n 56,\n 119,\n 99,\n 74,\n 120,\n 116,\n 54,\n 72,\n 69,\n 51,\n 88,\n 119,\n 83,\n 69,\n 117,\n 68,\n 75,\n 82,\n 111,\n 87,\n 77,\n 68,\n 106,\n 90,\n 119,\n 115,\n 79,\n 70,\n 102,\n 111,\n 107,\n 87,\n 119,\n 77,\n 66,\n 113,\n 51,\n 101,\n 49,\n 120,\n 84,\n 122,\n 90,\n 99,\n 10,\n 90,\n 102,\n 54,\n 48,\n 57,\n 70,\n 53,\n 53,\n 83,\n 105,\n 78,\n 107,\n 76,\n 83,\n 66,\n 100,\n 116,\n 112,\n 107,\n 51,\n 107,\n 105,\n 53,\n 115,\n 77,\n 70,\n 100,\n 113,\n 107,\n 99,\n 117,\n 86,\n 51,\n 78,\n 99,\n 84,\n 98,\n 85,\n 121,\n 90,\n 89,\n 43,\n 85,\n 71,\n 110,\n 111,\n 117,\n 47,\n 84,\n 72,\n 73,\n 101,\n 69,\n 53,\n 118,\n 111,\n 77,\n 107,\n 97,\n 114,\n 47,\n 114,\n 47,\n 50,\n 10,\n 68,\n 57,\n 49,\n 122,\n 112,\n 66,\n 98,\n 54,\n 109,\n 82,\n 100,\n 77,\n 82,\n 75,\n 48,\n 87,\n 102,\n 98,\n 106,\n 82,\n 120,\n 115,\n 119,\n 121,\n 86,\n 107,\n 74,\n 49,\n 98,\n 122,\n 90,\n 104,\n 110,\n 90,\n 54,\n 119,\n 88,\n 115,\n 98,\n 99,\n 55,\n 118,\n 56,\n 101,\n 56,\n 52,\n 88,\n 110,\n 56,\n 55,\n 70,\n 47,\n 106,\n 49,\n 101,\n 104,\n 82,\n 76,\n 121,\n 109,\n 87,\n 76,\n 104,\n 79,\n 10,\n 104,\n 120,\n 97,\n 50,\n 98,\n 54,\n 107,\n 55,\n 87,\n 90,\n 81,\n 89,\n 117,\n 78,\n 84,\n 49,\n 86,\n 108,\n 75,\n 51,\n 116,\n 66,\n 103,\n 109,\n 83,\n 90,\n 66,\n 53,\n 57,\n 85,\n 55,\n 86,\n 104,\n 121,\n 102,\n 54,\n 65,\n 74,\n 68,\n 43,\n 57,\n 75,\n 51,\n 56,\n 107,\n 97,\n 78,\n 49,\n 120,\n 71,\n 54,\n 65,\n 66,\n 90,\n 89,\n 121,\n 72,\n 66,\n 99,\n 101,\n 102,\n 66,\n 102,\n 51,\n 10,\n 43,\n 65,\n 90,\n 116,\n 113,\n 83,\n 83,\n 75,\n 79,\n 55,\n 52,\n 52,\n 108,\n 113,\n 88,\n 72,\n 84,\n 105,\n 109,\n 77,\n 121,\n 106,\n 122,\n 75,\n 106,\n 80,\n 78,\n 48,\n 120,\n 50,\n 65,\n 79,\n 73,\n 102,\n 112,\n 43,\n 66,\n 81,\n 50,\n 82,\n 55,\n 111,\n 72,\n 79,\n 83,\n 70,\n 54,\n 50,\n 47,\n 89,\n 101,\n 69,\n 86,\n 82,\n 103,\n 82,\n 70,\n 49,\n 103,\n 66,\n 65,\n 84,\n 51,\n 65,\n 10,\n 70,\n 102,\n 121,\n 68,\n 114,\n 110,\n 53,\n 77,\n 90,\n 55,\n 103,\n 75,\n 104,\n 117,\n 67,\n 48,\n 120,\n 67,\n 109,\n 107,\n 78,\n 57,\n 65,\n 87,\n 53,\n 118,\n 106,\n 51,\n 103,\n 108,\n 69,\n 121,\n 104,\n 71,\n 76,\n 72,\n 81,\n 53,\n 88,\n 67,\n 122,\n 79,\n 115,\n 54,\n 101,\n 102,\n 84,\n 118,\n 97,\n 86,\n 43,\n 57,\n 104,\n 121,\n 53,\n 101,\n 73,\n 87,\n 106,\n 116,\n 105,\n 102,\n 107,\n 98,\n 10,\n 122,\n 90,\n 77,\n 47,\n 80,\n 43,\n 48,\n 53,\n 100,\n 121,\n 99,\n 85,\n 78,\n 106,\n 49,\n 121,\n 119,\n 108,\n 73,\n 106,\n 117,\n 55,\n 97,\n 69,\n 48,\n 116,\n 69,\n 81,\n 75,\n 89,\n 48,\n 84,\n 49,\n 116,\n 103,\n 106,\n 121,\n 52,\n 73,\n 103,\n 43,\n 109,\n 43,\n 75,\n 110,\n 72,\n 70,\n 54,\n 108,\n 83,\n 106,\n 115,\n 70,\n 110,\n 50,\n 82,\n 75,\n 84,\n 56,\n 89,\n 57,\n 118,\n 81,\n 67,\n 10,\n 106,\n 100,\n 82,\n 121,\n 112,\n 97,\n 56,\n 79,\n 56,\n 53,\n 57,\n 98,\n 48,\n 51,\n 90,\n 52,\n 109,\n 89,\n 72,\n 82,\n 102,\n 99,\n 54,\n 102,\n 43,\n 72,\n 119,\n 112,\n 49,\n 121,\n 86,\n 105,\n 55,\n 71,\n 56,\n 69,\n 70,\n 112,\n 99,\n 108,\n 83,\n 85,\n 54,\n 79,\n 69,\n 81,\n 111,\n 101,\n 75,\n 70,\n 56,\n 48,\n 84,\n 66,\n 71,\n 111,\n 10,\n 45,\n 45,\n 45,\n 45,\n 45,\n 69,\n 78,\n 68,\n 32,\n 67,\n 69,\n 82,\n 84,\n 73,\n 70,\n 73,\n 67,\n 65,\n 84,\n 69,\n 45,\n 45,\n 45,\n 45,\n 45,\n 10,\n ],\n key: [\n 45,\n 45,\n 45,\n 45,\n 45,\n 66,\n 69,\n 71,\n 73,\n 78,\n 32,\n 80,\n 82,\n 73,\n 86,\n 65,\n 84,\n 69,\n 32,\n 75,\n 69,\n 89,\n 45,\n 45,\n 45,\n 45,\n 45,\n 10,\n 77,\n 73,\n 73,\n 74,\n 81,\n 81,\n 73,\n 66,\n 65,\n 68,\n 65,\n 78,\n 66,\n 103,\n 107,\n 113,\n 104,\n 107,\n 105,\n 71,\n 57,\n 119,\n 48,\n 66,\n 65,\n 81,\n 69,\n 70,\n 65,\n 65,\n 83,\n 67,\n 67,\n 83,\n 115,\n 119,\n 103,\n 103,\n 107,\n 110,\n 65,\n 103,\n 69,\n 65,\n 65,\n 111,\n 73,\n 67,\n 65,\n 81,\n 68,\n 67,\n 76,\n 85,\n 85,\n 73,\n 83,\n 88,\n 103,\n 106,\n 71,\n 67,\n 49,\n 74,\n 10,\n 86,\n 47,\n 109,\n 80,\n 90,\n 106,\n 83,\n 98,\n 118,\n 56,\n 76,\n 69,\n 110,\n 97,\n 78,\n 97,\n 52,\n 49,\n 119,\n 57,\n 120,\n 101,\n 102,\n 122,\n 118,\n 105,\n 119,\n 105,\n 74,\n 108,\n 80,\n 87,\n 77,\n 105,\n 114,\n 69,\n 68,\n 73,\n 101,\n 118,\n 67,\n 104,\n 85,\n 104,\n 43,\n 115,\n 104,\n 67,\n 51,\n 77,\n 56,\n 86,\n 120,\n 53,\n 99,\n 72,\n 116,\n 117,\n 101,\n 78,\n 112,\n 52,\n 74,\n 75,\n 10,\n 84,\n 48,\n 120,\n 120,\n 87,\n 100,\n 118,\n 57,\n 117,\n 121,\n 87,\n 87,\n 72,\n 107,\n 74,\n 113,\n 74,\n 57,\n 122,\n 105,\n 118,\n 112,\n 71,\n 56,\n 122,\n 117,\n 51,\n 122,\n 107,\n 112,\n 78,\n 101,\n 113,\n 113,\n 83,\n 79,\n 122,\n 115,\n 104,\n 120,\n 119,\n 99,\n 54,\n 86,\n 57,\n 79,\n 74,\n 57,\n 43,\n 83,\n 85,\n 107,\n 113,\n 117,\n 75,\n 50,\n 103,\n 122,\n 54,\n 54,\n 48,\n 69,\n 117,\n 53,\n 10,\n 117,\n 74,\n 108,\n 113,\n 97,\n 97,\n 108,\n 108,\n 112,\n 66,\n 65,\n 57,\n 101,\n 80,\n 57,\n 83,\n 113,\n 53,\n 114,\n 102,\n 49,\n 110,\n 43,\n 49,\n 51,\n 74,\n 117,\n 102,\n 101,\n 108,\n 76,\n 106,\n 117,\n 84,\n 65,\n 78,\n 111,\n 118,\n 57,\n 79,\n 55,\n 57,\n 49,\n 73,\n 88,\n 85,\n 77,\n 69,\n 122,\n 43,\n 74,\n 48,\n 84,\n 83,\n 52,\n 77,\n 101,\n 65,\n 89,\n 56,\n 106,\n 51,\n 112,\n 50,\n 10,\n 82,\n 106,\n 85,\n 99,\n 84,\n 82,\n 111,\n 120,\n 122,\n 119,\n 115,\n 100,\n 114,\n 89,\n 53,\n 66,\n 104,\n 68,\n 122,\n 48,\n 103,\n 67,\n 120,\n 53,\n 116,\n 79,\n 65,\n 66,\n 76,\n 83,\n 88,\n 71,\n 116,\n 85,\n 78,\n 76,\n 103,\n 68,\n 105,\n 49,\n 90,\n 117,\n 106,\n 53,\n 81,\n 76,\n 74,\n 51,\n 107,\n 98,\n 70,\n 71,\n 112,\n 66,\n 100,\n 109,\n 104,\n 120,\n 74,\n 119,\n 98,\n 66,\n 98,\n 53,\n 10,\n 75,\n 110,\n 80,\n 115,\n 109,\n 90,\n 113,\n 70,\n 85,\n 112,\n 112,\n 72,\n 122,\n 109,\n 75,\n 65,\n 76,\n 111,\n 121,\n 68,\n 86,\n 75,\n 120,\n 70,\n 52,\n 49,\n 105,\n 65,\n 101,\n 115,\n 117,\n 90,\n 75,\n 78,\n 99,\n 67,\n 119,\n 83,\n 70,\n 119,\n 108,\n 72,\n 104,\n 100,\n 54,\n 109,\n 88,\n 49,\n 47,\n 55,\n 122,\n 48,\n 80,\n 50,\n 78,\n 83,\n 98,\n 43,\n 112,\n 76,\n 99,\n 110,\n 49,\n 109,\n 10,\n 89,\n 65,\n 107,\n 121,\n 114,\n 101,\n 57,\n 90,\n 107,\n 88,\n 78,\n 102,\n 121,\n 78,\n 117,\n 99,\n 73,\n 78,\n 43,\n 77,\n 43,\n 52,\n 97,\n 52,\n 103,\n 48,\n 71,\n 121,\n 55,\n 75,\n 119,\n 82,\n 76,\n 102,\n 106,\n 119,\n 85,\n 70,\n 79,\n 66,\n 88,\n 99,\n 49,\n 50,\n 79,\n 120,\n 48,\n 117,\n 120,\n 70,\n 48,\n 74,\n 89,\n 114,\n 117,\n 50,\n 117,\n 103,\n 100,\n 84,\n 90,\n 89,\n 50,\n 74,\n 10,\n 52,\n 78,\n 88,\n 90,\n 121,\n 89,\n 53,\n 43,\n 76,\n 84,\n 115,\n 79,\n 111,\n 89,\n 90,\n 47,\n 115,\n 53,\n 114,\n 102,\n 101,\n 99,\n 111,\n 114,\n 50,\n 76,\n 101,\n 81,\n 109,\n 104,\n 98,\n 112,\n 57,\n 87,\n 81,\n 66,\n 82,\n 120,\n 69,\n 86,\n 65,\n 53,\n 56,\n 77,\n 117,\n 89,\n 121,\n 103,\n 100,\n 80,\n 48,\n 79,\n 122,\n 51,\n 50,\n 52,\n 119,\n 79,\n 85,\n 67,\n 52,\n 56,\n 57,\n 120,\n 10,\n 77,\n 69,\n 54,\n 74,\n 83,\n 80,\n 81,\n 112,\n 53,\n 110,\n 52,\n 116,\n 67,\n 57,\n 117,\n 52,\n 113,\n 109,\n 86,\n 74,\n 76,\n 88,\n 101,\n 122,\n 52,\n 99,\n 73,\n 107,\n 43,\n 82,\n 121,\n 79,\n 120,\n 118,\n 118,\n 49,\n 97,\n 111,\n 57,\n 79,\n 121,\n 90,\n 69,\n 108,\n 67,\n 118,\n 54,\n 57,\n 57,\n 105,\n 89,\n 118,\n 107,\n 54,\n 65,\n 100,\n 51,\n 67,\n 89,\n 105,\n 121,\n 82,\n 52,\n 78,\n 10,\n 73,\n 56,\n 52,\n 71,\n 122,\n 122,\n 81,\n 76,\n 50,\n 55,\n 81,\n 57,\n 102,\n 100,\n 88,\n 90,\n 85,\n 112,\n 76,\n 83,\n 99,\n 98,\n 86,\n 99,\n 88,\n 112,\n 43,\n 57,\n 116,\n 103,\n 82,\n 69,\n 104,\n 104,\n 69,\n 73,\n 102,\n 70,\n 50,\n 114,\n 112,\n 68,\n 72,\n 56,\n 73,\n 104,\n 71,\n 116,\n 103,\n 110,\n 49,\n 98,\n 56,\n 81,\n 50,\n 113,\n 47,\n 119,\n 98,\n 88,\n 112,\n 112,\n 122,\n 69,\n 10,\n 55,\n 99,\n 43,\n 79,\n 85,\n 121,\n 113,\n 107,\n 104,\n 57,\n 97,\n 84,\n 57,\n 110,\n 112,\n 74,\n 69,\n 122,\n 85,\n 56,\n 98,\n 43,\n 68,\n 105,\n 52,\n 43,\n 117,\n 119,\n 86,\n 65,\n 70,\n 100,\n 107,\n 56,\n 84,\n 76,\n 65,\n 87,\n 121,\n 54,\n 110,\n 89,\n 79,\n 87,\n 74,\n 97,\n 118,\n 55,\n 120,\n 89,\n 103,\n 74,\n 122,\n 113,\n 88,\n 85,\n 99,\n 73,\n 50,\n 82,\n 108,\n 49,\n 82,\n 121,\n 10,\n 117,\n 113,\n 79,\n 81,\n 80,\n 86,\n 111,\n 118,\n 49,\n 43,\n 108,\n 97,\n 50,\n 74,\n 76,\n 53,\n 80,\n 48,\n 66,\n 117,\n 82,\n 117,\n 72,\n 119,\n 120,\n 110,\n 80,\n 75,\n 85,\n 119,\n 73,\n 68,\n 65,\n 81,\n 65,\n 66,\n 65,\n 111,\n 73,\n 67,\n 65,\n 72,\n 88,\n 122,\n 106,\n 100,\n 87,\n 69,\n 50,\n 52,\n 100,\n 108,\n 55,\n 107,\n 120,\n 53,\n 78,\n 107,\n 49,\n 88,\n 90,\n 109,\n 109,\n 100,\n 10,\n 121,\n 76,\n 65,\n 113,\n 109,\n 99,\n 112,\n 105,\n 66,\n 101,\n 119,\n 52,\n 82,\n 98,\n 103,\n 89,\n 77,\n 122,\n 76,\n 106,\n 87,\n 86,\n 88,\n 112,\n 122,\n 51,\n 83,\n 98,\n 114,\n 43,\n 103,\n 121,\n 70,\n 98,\n 103,\n 52,\n 89,\n 71,\n 72,\n 78,\n 81,\n 55,\n 89,\n 86,\n 77,\n 98,\n 99,\n 114,\n 56,\n 101,\n 104,\n 111,\n 99,\n 111,\n 113,\n 49,\n 78,\n 78,\n 103,\n 43,\n 72,\n 53,\n 89,\n 51,\n 10,\n 107,\n 112,\n 78,\n 110,\n 43,\n 105,\n 119,\n 70,\n 51,\n 118,\n 87,\n 86,\n 107,\n 87,\n 51,\n 66,\n 118,\n 87,\n 87,\n 83,\n 67,\n 104,\n 99,\n 100,\n 84,\n 83,\n 121,\n 54,\n 105,\n 54,\n 81,\n 101,\n 101,\n 71,\n 69,\n 55,\n 67,\n 107,\n 73,\n 47,\n 106,\n 56,\n 75,\n 65,\n 85,\n 52,\n 116,\n 79,\n 109,\n 108,\n 74,\n 99,\n 113,\n 85,\n 77,\n 57,\n 48,\n 87,\n 79,\n 69,\n 110,\n 55,\n 80,\n 119,\n 10,\n 80,\n 122,\n 85,\n 52,\n 82,\n 78,\n 66,\n 120,\n 69,\n 88,\n 107,\n 98,\n 69,\n 83,\n 84,\n 108,\n 120,\n 49,\n 114,\n 79,\n 115,\n 88,\n 57,\n 72,\n 82,\n 78,\n 116,\n 106,\n 119,\n 102,\n 97,\n 118,\n 87,\n 104,\n 110,\n 86,\n 72,\n 82,\n 114,\n 86,\n 78,\n 78,\n 103,\n 48,\n 56,\n 100,\n 51,\n 57,\n 73,\n 114,\n 121,\n 121,\n 52,\n 48,\n 55,\n 112,\n 81,\n 117,\n 88,\n 89,\n 75,\n 67,\n 116,\n 78,\n 10,\n 101,\n 53,\n 69,\n 104,\n 108,\n 105,\n 74,\n 83,\n 77,\n 55,\n 122,\n 75,\n 66,\n 122,\n 111,\n 47,\n 70,\n 122,\n 116,\n 102,\n 115,\n 50,\n 122,\n 76,\n 79,\n 118,\n 43,\n 110,\n 43,\n 99,\n 113,\n 110,\n 111,\n 113,\n 109,\n 87,\n 70,\n 86,\n 76,\n 119,\n 109,\n 50,\n 69,\n 118,\n 50,\n 69,\n 56,\n 100,\n 105,\n 97,\n 80,\n 107,\n 53,\n 79,\n 101,\n 105,\n 86,\n 66,\n 122,\n 76,\n 67,\n 99,\n 81,\n 109,\n 10,\n 56,\n 122,\n 84,\n 102,\n 78,\n 53,\n 74,\n 112,\n 116,\n 69,\n 48,\n 113,\n 113,\n 67,\n 77,\n 107,\n 65,\n 79,\n 83,\n 54,\n 115,\n 117,\n 90,\n 115,\n 109,\n 110,\n 56,\n 47,\n 49,\n 104,\n 74,\n 79,\n 47,\n 102,\n 98,\n 43,\n 112,\n 99,\n 85,\n 83,\n 47,\n 106,\n 76,\n 89,\n 78,\n 110,\n 115,\n 106,\n 74,\n 73,\n 50,\n 69,\n 70,\n 73,\n 121,\n 86,\n 108,\n 99,\n 68,\n 70,\n 83,\n 51,\n 104,\n 114,\n 10,\n 97,\n 107,\n 89,\n 115,\n 116,\n 112,\n 80,\n 75,\n 110,\n 77,\n 77,\n 122,\n 55,\n 77,\n 72,\n 98,\n 71,\n 76,\n 112,\n 67,\n 82,\n 85,\n 121,\n 43,\n 81,\n 107,\n 49,\n 79,\n 117,\n 120,\n 88,\n 119,\n 65,\n 79,\n 74,\n 116,\n 104,\n 73,\n 113,\n 48,\n 84,\n 85,\n 90,\n 105,\n 90,\n 90,\n 57,\n 71,\n 74,\n 84,\n 100,\n 120,\n 99,\n 49,\n 102,\n 51,\n 115,\n 88,\n 80,\n 119,\n 103,\n 119,\n 56,\n 54,\n 10,\n 117,\n 71,\n 98,\n 119,\n 55,\n 67,\n 53,\n 84,\n 120,\n 84,\n 98,\n 43,\n 48,\n 103,\n 113,\n 74,\n 43,\n 49,\n 43,\n 73,\n 70,\n 77,\n 111,\n 75,\n 113,\n 51,\n 77,\n 88,\n 87,\n 69,\n 110,\n 72,\n 69,\n 98,\n 99,\n 86,\n 119,\n 90,\n 119,\n 53,\n 78,\n 106,\n 76,\n 57,\n 110,\n 65,\n 115,\n 83,\n 111,\n 81,\n 69,\n 43,\n 80,\n 122,\n 84,\n 87,\n 71,\n 104,\n 110,\n 100,\n 86,\n 70,\n 67,\n 119,\n 10,\n 65,\n 67,\n 98,\n 43,\n 89,\n 113,\n 71,\n 116,\n 79,\n 69,\n 104,\n 75,\n 68,\n 80,\n 70,\n 116,\n 49,\n 117,\n 80,\n 103,\n 88,\n 81,\n 89,\n 98,\n 68,\n 115,\n 104,\n 86,\n 68,\n 70,\n 101,\n 82,\n 118,\n 119,\n 114,\n 84,\n 85,\n 115,\n 113,\n 83,\n 83,\n 65,\n 109,\n 119,\n 120,\n 86,\n 65,\n 43,\n 98,\n 89,\n 98,\n 55,\n 120,\n 73,\n 114,\n 100,\n 105,\n 85,\n 107,\n 98,\n 71,\n 65,\n 43,\n 115,\n 10,\n 118,\n 69,\n 90,\n 43,\n 66,\n 99,\n 108,\n 76,\n 56,\n 82,\n 71,\n 53,\n 102,\n 67,\n 113,\n 112,\n 119,\n 111,\n 53,\n 57,\n 101,\n 113,\n 47,\n 117,\n 47,\n 81,\n 105,\n 115,\n 116,\n 77,\n 113,\n 111,\n 81,\n 114,\n 72,\n 82,\n 85,\n 65,\n 72,\n 98,\n 83,\n 100,\n 55,\n 49,\n 81,\n 119,\n 116,\n 111,\n 48,\n 81,\n 86,\n 68,\n 85,\n 116,\n 47,\n 65,\n 66,\n 75,\n 71,\n 77,\n 98,\n 69,\n 78,\n 57,\n 10,\n 51,\n 104,\n 75,\n 50,\n 105,\n 102,\n 73,\n 84,\n 102,\n 97,\n 75,\n 102,\n 111,\n 111,\n 107,\n 72,\n 82,\n 73,\n 116,\n 86,\n 72,\n 50,\n 79,\n 90,\n 103,\n 81,\n 83,\n 43,\n 122,\n 49,\n 68,\n 106,\n 53,\n 89,\n 67,\n 115,\n 116,\n 76,\n 107,\n 78,\n 111,\n 78,\n 79,\n 99,\n 69,\n 99,\n 43,\n 103,\n 80,\n 55,\n 55,\n 67,\n 108,\n 43,\n 106,\n 67,\n 87,\n 54,\n 98,\n 82,\n 113,\n 48,\n 78,\n 114,\n 10,\n 110,\n 118,\n 80,\n 114,\n 118,\n 110,\n 118,\n 43,\n 106,\n 104,\n 84,\n 72,\n 108,\n 103,\n 83,\n 50,\n 106,\n 57,\n 84,\n 104,\n 65,\n 111,\n 73,\n 66,\n 65,\n 81,\n 68,\n 119,\n 115,\n 100,\n 66,\n 103,\n 75,\n 105,\n 51,\n 115,\n 107,\n 49,\n 117,\n 51,\n 68,\n 121,\n 56,\n 98,\n 74,\n 108,\n 122,\n 101,\n 52,\n 114,\n 89,\n 119,\n 109,\n 88,\n 100,\n 118,\n 75,\n 65,\n 105,\n 68,\n 70,\n 83,\n 54,\n 70,\n 10,\n 118,\n 53,\n 114,\n 56,\n 66,\n 84,\n 84,\n 54,\n 104,\n 54,\n 66,\n 112,\n 53,\n 106,\n 87,\n 65,\n 105,\n 108,\n 87,\n 57,\n 72,\n 48,\n 85,\n 72,\n 89,\n 71,\n 86,\n 72,\n 113,\n 73,\n 71,\n 53,\n 51,\n 54,\n 97,\n 111,\n 113,\n 49,\n 81,\n 49,\n 75,\n 84,\n 86,\n 72,\n 105,\n 89,\n 84,\n 82,\n 102,\n 81,\n 121,\n 55,\n 73,\n 100,\n 118,\n 120,\n 89,\n 75,\n 48,\n 54,\n 106,\n 108,\n 79,\n 117,\n 10,\n 49,\n 102,\n 99,\n 103,\n 99,\n 50,\n 52,\n 117,\n 69,\n 122,\n 56,\n 106,\n 116,\n 115,\n 55,\n 76,\n 115,\n 90,\n 68,\n 47,\n 47,\n 116,\n 73,\n 43,\n 69,\n 73,\n 69,\n 118,\n 103,\n 116,\n 107,\n 98,\n 80,\n 80,\n 119,\n 90,\n 82,\n 121,\n 121,\n 100,\n 119,\n 51,\n 108,\n 79,\n 113,\n 98,\n 85,\n 74,\n 84,\n 71,\n 49,\n 84,\n 48,\n 78,\n 74,\n 89,\n 97,\n 78,\n 84,\n 111,\n 108,\n 86,\n 68,\n 50,\n 10,\n 77,\n 115,\n 84,\n 50,\n 117,\n 85,\n 87,\n 100,\n 117,\n 53,\n 120,\n 49,\n 115,\n 113,\n 103,\n 111,\n 87,\n 120,\n 100,\n 98,\n 43,\n 71,\n 104,\n 87,\n 57,\n 116,\n 51,\n 112,\n 100,\n 100,\n 100,\n 83,\n 103,\n 88,\n 56,\n 99,\n 74,\n 120,\n 77,\n 102,\n 120,\n 103,\n 80,\n 83,\n 65,\n 99,\n 105,\n 100,\n 80,\n 86,\n 110,\n 119,\n 101,\n 98,\n 88,\n 118,\n 121,\n 113,\n 81,\n 86,\n 74,\n 110,\n 85,\n 66,\n 10,\n 112,\n 118,\n 67,\n 101,\n 97,\n 104,\n 104,\n 114,\n 52,\n 89,\n 119,\n 121,\n 120,\n 109,\n 90,\n 107,\n 104,\n 114,\n 78,\n 115,\n 70,\n 75,\n 71,\n 78,\n 114,\n 110,\n 90,\n 111,\n 73,\n 84,\n 105,\n 85,\n 122,\n 78,\n 48,\n 50,\n 43,\n 56,\n 82,\n 82,\n 97,\n 119,\n 103,\n 87,\n 101,\n 53,\n 89,\n 102,\n 113,\n 104,\n 117,\n 106,\n 72,\n 89,\n 79,\n 100,\n 75,\n 87,\n 102,\n 48,\n 52,\n 83,\n 117,\n 71,\n 10,\n 111,\n 118,\n 84,\n 109,\n 74,\n 107,\n 66,\n 89,\n 112,\n 99,\n 53,\n 109,\n 89,\n 100,\n 77,\n 119,\n 88,\n 106,\n 104,\n 120,\n 84,\n 108,\n 83,\n 69,\n 79,\n 70,\n 48,\n 89,\n 113,\n 113,\n 109,\n 57,\n 69,\n 56,\n 57,\n 105,\n 69,\n 98,\n 79,\n 82,\n 121,\n 87,\n 114,\n 99,\n 108,\n 89,\n 65,\n 76,\n 65,\n 111,\n 73,\n 66,\n 65,\n 81,\n 68,\n 79,\n 104,\n 106,\n 86,\n 54,\n 86,\n 87,\n 116,\n 68,\n 10,\n 113,\n 98,\n 55,\n 71,\n 67,\n 117,\n 48,\n 101,\n 111,\n 75,\n 118,\n 85,\n 116,\n 106,\n 122,\n 66,\n 107,\n 122,\n 80,\n 73,\n 102,\n 75,\n 74,\n 87,\n 86,\n 100,\n 110,\n 50,\n 85,\n 104,\n 100,\n 69,\n 50,\n 102,\n 100,\n 51,\n 55,\n 114,\n 113,\n 55,\n 68,\n 97,\n 88,\n 82,\n 115,\n 47,\n 85,\n 87,\n 101,\n 70,\n 75,\n 113,\n 90,\n 76,\n 117,\n 97,\n 108,\n 80,\n 72,\n 98,\n 72,\n 105,\n 111,\n 56,\n 10,\n 76,\n 119,\n 119,\n 83,\n 87,\n 82,\n 122,\n 68,\n 114,\n 53,\n 108,\n 83,\n 121,\n 112,\n 103,\n 84,\n 116,\n 69,\n 65,\n 47,\n 100,\n 77,\n 122,\n 71,\n 87,\n 116,\n 112,\n 111,\n 75,\n 54,\n 118,\n 53,\n 98,\n 85,\n 115,\n 78,\n 49,\n 49,\n 79,\n 120,\n 57,\n 113,\n 104,\n 67,\n 113,\n 73,\n 89,\n 119,\n 57,\n 104,\n 89,\n 87,\n 56,\n 54,\n 118,\n 71,\n 68,\n 47,\n 110,\n 49,\n 49,\n 77,\n 113,\n 87,\n 10,\n 72,\n 118,\n 65,\n 86,\n 53,\n 115,\n 49,\n 78,\n 80,\n 113,\n 122,\n 84,\n 99,\n 116,\n 73,\n 73,\n 79,\n 81,\n 115,\n 122,\n 113,\n 88,\n 90,\n 97,\n 116,\n 99,\n 72,\n 117,\n 107,\n 67,\n 54,\n 102,\n 73,\n 87,\n 116,\n 106,\n 113,\n 120,\n 122,\n 100,\n 52,\n 88,\n 72,\n 122,\n 104,\n 112,\n 89,\n 72,\n 102,\n 118,\n 71,\n 113,\n 90,\n 120,\n 65,\n 110,\n 112,\n 76,\n 65,\n 49,\n 97,\n 83,\n 112,\n 47,\n 10,\n 54,\n 50,\n 56,\n 48,\n 97,\n 50,\n 67,\n 87,\n 103,\n 121,\n 67,\n 111,\n 78,\n 89,\n 48,\n 99,\n 114,\n 89,\n 75,\n 80,\n 114,\n 87,\n 76,\n 81,\n 100,\n 102,\n 67,\n 106,\n 107,\n 49,\n 72,\n 110,\n 98,\n 119,\n 110,\n 49,\n 112,\n 72,\n 55,\n 115,\n 75,\n 53,\n 70,\n 47,\n 113,\n 103,\n 113,\n 76,\n 68,\n 86,\n 68,\n 90,\n 103,\n 116,\n 69,\n 112,\n 118,\n 105,\n 48,\n 75,\n 65,\n 120,\n 74,\n 57,\n 10,\n 47,\n 106,\n 52,\n 47,\n 114,\n 53,\n 119,\n 78,\n 56,\n 54,\n 52,\n 112,\n 71,\n 56,\n 85,\n 52,\n 90,\n 115,\n 100,\n 109,\n 77,\n 82,\n 90,\n 53,\n 85,\n 79,\n 76,\n 111,\n 51,\n 43,\n 108,\n 88,\n 88,\n 83,\n 89,\n 108,\n 73,\n 104,\n 55,\n 77,\n 66,\n 71,\n 118,\n 66,\n 77,\n 79,\n 68,\n 102,\n 50,\n 48,\n 79,\n 86,\n 55,\n 107,\n 103,\n 109,\n 100,\n 122,\n 71,\n 76,\n 56,\n 69,\n 113,\n 87,\n 10,\n 117,\n 112,\n 43,\n 112,\n 112,\n 80,\n 66,\n 70,\n 89,\n 109,\n 80,\n 90,\n 65,\n 111,\n 73,\n 66,\n 65,\n 72,\n 72,\n 56,\n 56,\n 57,\n 56,\n 109,\n 43,\n 112,\n 114,\n 73,\n 97,\n 113,\n 51,\n 50,\n 72,\n 101,\n 118,\n 119,\n 114,\n 104,\n 106,\n 77,\n 75,\n 49,\n 89,\n 50,\n 120,\n 56,\n 52,\n 51,\n 109,\n 108,\n 104,\n 104,\n 53,\n 113,\n 100,\n 109,\n 120,\n 52,\n 74,\n 70,\n 65,\n 81,\n 77,\n 73,\n 10,\n 121,\n 104,\n 112,\n 56,\n 56,\n 75,\n 101,\n 81,\n 87,\n 43,\n 120,\n 49,\n 54,\n 53,\n 52,\n 68,\n 115,\n 57,\n 78,\n 77,\n 53,\n 83,\n 114,\n 101,\n 49,\n 84,\n 71,\n 90,\n 43,\n 103,\n 89,\n 51,\n 72,\n 115,\n 108,\n 106,\n 81,\n 105,\n 104,\n 110,\n 51,\n 87,\n 105,\n 101,\n 76,\n 50,\n 104,\n 80,\n 122,\n 90,\n 111,\n 109,\n 110,\n 49,\n 68,\n 48,\n 101,\n 100,\n 87,\n 47,\n 100,\n 78,\n 87,\n 84,\n 10,\n 80,\n 100,\n 113,\n 118,\n 68,\n 53,\n 53,\n 57,\n 50,\n 70,\n 113,\n 48,\n 70,\n 109,\n 78,\n 89,\n 77,\n 54,\n 89,\n 74,\n 104,\n 77,\n 73,\n 87,\n 106,\n 112,\n 108,\n 50,\n 69,\n 74,\n 89,\n 105,\n 90,\n 99,\n 101,\n 119,\n 76,\n 86,\n 70,\n 98,\n 66,\n 89,\n 101,\n 108,\n 74,\n 73,\n 106,\n 69,\n 70,\n 78,\n 105,\n 86,\n 74,\n 107,\n 79,\n 67,\n 90,\n 97,\n 56,\n 109,\n 114,\n 79,\n 90,\n 51,\n 10,\n 88,\n 68,\n 67,\n 78,\n 84,\n 72,\n 47,\n 89,\n 75,\n 43,\n 98,\n 100,\n 98,\n 75,\n 83,\n 75,\n 119,\n 121,\n 116,\n 50,\n 75,\n 71,\n 81,\n 108,\n 101,\n 99,\n 111,\n 101,\n 89,\n 122,\n 115,\n 118,\n 55,\n 115,\n 48,\n 57,\n 70,\n 81,\n 73,\n 75,\n 100,\n 82,\n 97,\n 105,\n 118,\n 86,\n 118,\n 122,\n 106,\n 117,\n 79,\n 74,\n 85,\n 43,\n 53,\n 86,\n 88,\n 105,\n 122,\n 88,\n 116,\n 47,\n 115,\n 88,\n 10,\n 86,\n 79,\n 115,\n 83,\n 57,\n 74,\n 106,\n 49,\n 66,\n 102,\n 72,\n 81,\n 106,\n 51,\n 88,\n 109,\n 119,\n 113,\n 75,\n 114,\n 52,\n 80,\n 122,\n 118,\n 84,\n 72,\n 66,\n 109,\n 113,\n 87,\n 71,\n 90,\n 109,\n 55,\n 110,\n 121,\n 67,\n 51,\n 87,\n 120,\n 102,\n 79,\n 99,\n 120,\n 54,\n 103,\n 117,\n 54,\n 43,\n 85,\n 87,\n 75,\n 103,\n 97,\n 43,\n 65,\n 97,\n 120,\n 83,\n 51,\n 49,\n 115,\n 57,\n 107,\n 10,\n 114,\n 113,\n 118,\n 106,\n 57,\n 106,\n 48,\n 52,\n 48,\n 101,\n 103,\n 118,\n 72,\n 87,\n 50,\n 69,\n 116,\n 43,\n 111,\n 102,\n 111,\n 77,\n 85,\n 118,\n 112,\n 89,\n 52,\n 102,\n 76,\n 102,\n 105,\n 68,\n 57,\n 47,\n 122,\n 66,\n 80,\n 71,\n 77,\n 67,\n 103,\n 103,\n 69,\n 65,\n 77,\n 104,\n 117,\n 48,\n 67,\n 47,\n 116,\n 47,\n 114,\n 65,\n 111,\n 97,\n 90,\n 72,\n 105,\n 119,\n 77,\n 90,\n 99,\n 119,\n 10,\n 79,\n 99,\n 115,\n 66,\n 107,\n 77,\n 73,\n 68,\n 100,\n 87,\n 104,\n 119,\n 66,\n 116,\n 121,\n 76,\n 112,\n 115,\n 48,\n 105,\n 73,\n 105,\n 49,\n 48,\n 99,\n 112,\n 47,\n 48,\n 112,\n 72,\n 47,\n 97,\n 78,\n 76,\n 103,\n 105,\n 66,\n 104,\n 75,\n 112,\n 107,\n 47,\n 103,\n 101,\n 52,\n 74,\n 71,\n 48,\n 52,\n 77,\n 73,\n 47,\n 52,\n 120,\n 120,\n 84,\n 74,\n 72,\n 121,\n 69,\n 107,\n 121,\n 73,\n 101,\n 10,\n 53,\n 69,\n 121,\n 78,\n 101,\n 51,\n 74,\n 110,\n 105,\n 68,\n 51,\n 120,\n 108,\n 85,\n 114,\n 106,\n 53,\n 82,\n 82,\n 55,\n 104,\n 71,\n 103,\n 49,\n 74,\n 113,\n 87,\n 82,\n 66,\n 65,\n 80,\n 104,\n 114,\n 86,\n 75,\n 119,\n 82,\n 70,\n 55,\n 103,\n 50,\n 121,\n 55,\n 67,\n 66,\n 97,\n 116,\n 73,\n 114,\n 90,\n 71,\n 48,\n 109,\n 90,\n 110,\n 109,\n 68,\n 74,\n 112,\n 85,\n 81,\n 54,\n 75,\n 113,\n 10,\n 85,\n 104,\n 57,\n 119,\n 119,\n 70,\n 102,\n 43,\n 49,\n 77,\n 107,\n 81,\n 69,\n 49,\n 65,\n 52,\n 80,\n 111,\n 97,\n 122,\n 71,\n 82,\n 56,\n 70,\n 114,\n 69,\n 103,\n 80,\n 70,\n 47,\n 75,\n 107,\n 77,\n 68,\n 55,\n 43,\n 86,\n 100,\n 70,\n 67,\n 70,\n 50,\n 77,\n 115,\n 77,\n 48,\n 73,\n 116,\n 69,\n 114,\n 104,\n 121,\n 102,\n 70,\n 107,\n 71,\n 81,\n 85,\n 83,\n 119,\n 83,\n 98,\n 83,\n 70,\n 10,\n 68,\n 117,\n 114,\n 51,\n 85,\n 118,\n 98,\n 121,\n 84,\n 54,\n 80,\n 48,\n 55,\n 114,\n 107,\n 43,\n 54,\n 113,\n 70,\n 75,\n 53,\n 81,\n 107,\n 50,\n 57,\n 71,\n 101,\n 49,\n 57,\n 57,\n 49,\n 87,\n 84,\n 120,\n 66,\n 86,\n 119,\n 73,\n 107,\n 53,\n 106,\n 90,\n 122,\n 69,\n 51,\n 81,\n 106,\n 115,\n 97,\n 112,\n 112,\n 50,\n 79,\n 83,\n 87,\n 102,\n 117,\n 116,\n 111,\n 68,\n 49,\n 103,\n 114,\n 67,\n 10,\n 118,\n 101,\n 121,\n 106,\n 47,\n 70,\n 120,\n 76,\n 107,\n 119,\n 114,\n 106,\n 57,\n 103,\n 108,\n 102,\n 70,\n 76,\n 117,\n 72,\n 74,\n 82,\n 66,\n 80,\n 117,\n 103,\n 67,\n 43,\n 85,\n 105,\n 102,\n 99,\n 99,\n 83,\n 74,\n 72,\n 51,\n 87,\n 105,\n 89,\n 122,\n 70,\n 55,\n 108,\n 71,\n 74,\n 103,\n 103,\n 54,\n 71,\n 65,\n 106,\n 54,\n 78,\n 66,\n 99,\n 47,\n 115,\n 56,\n 114,\n 69,\n 78,\n 113,\n 102,\n 10,\n 101,\n 81,\n 75,\n 67,\n 65,\n 81,\n 65,\n 100,\n 90,\n 47,\n 56,\n 57,\n 119,\n 110,\n 118,\n 69,\n 106,\n 66,\n 73,\n 100,\n 114,\n 84,\n 80,\n 66,\n 101,\n 53,\n 89,\n 89,\n 56,\n 69,\n 110,\n 66,\n 71,\n 116,\n 108,\n 111,\n 122,\n 84,\n 111,\n 114,\n 108,\n 43,\n 74,\n 99,\n 81,\n 119,\n 105,\n 53,\n 115,\n 105,\n 82,\n 110,\n 97,\n 78,\n 57,\n 83,\n 74,\n 49,\n 76,\n 90,\n 103,\n 43,\n 82,\n 52,\n 10,\n 70,\n 51,\n 111,\n 87,\n 114,\n 107,\n 72,\n 121,\n 86,\n 119,\n 70,\n 99,\n 122,\n 72,\n 69,\n 120,\n 85,\n 88,\n 90,\n 118,\n 70,\n 69,\n 88,\n 97,\n 53,\n 68,\n 75,\n 71,\n 87,\n 48,\n 112,\n 66,\n 55,\n 117,\n 55,\n 105,\n 110,\n 99,\n 47,\n 104,\n 112,\n 120,\n 48,\n 86,\n 51,\n 120,\n 97,\n 98,\n 65,\n 71,\n 115,\n 83,\n 75,\n 116,\n 80,\n 113,\n 43,\n 121,\n 54,\n 116,\n 71,\n 114,\n 68,\n 53,\n 10,\n 111,\n 43,\n 57,\n 66,\n 87,\n 65,\n 57,\n 104,\n 115,\n 119,\n 113,\n 74,\n 66,\n 82,\n 75,\n 76,\n 118,\n 67,\n 119,\n 80,\n 106,\n 75,\n 74,\n 69,\n 84,\n 104,\n 48,\n 120,\n 90,\n 85,\n 52,\n 102,\n 98,\n 88,\n 119,\n 122,\n 69,\n 120,\n 84,\n 67,\n 78,\n 111,\n 79,\n 88,\n 74,\n 47,\n 105,\n 55,\n 100,\n 108,\n 110,\n 54,\n 55,\n 110,\n 86,\n 105,\n 87,\n 69,\n 68,\n 98,\n 73,\n 52,\n 113,\n 109,\n 10,\n 50,\n 49,\n 89,\n 122,\n 65,\n 83,\n 120,\n 68,\n 98,\n 109,\n 79,\n 55,\n 89,\n 106,\n 122,\n 77,\n 50,\n 84,\n 118,\n 57,\n 81,\n 78,\n 97,\n 82,\n 89,\n 88,\n 48,\n 77,\n 88,\n 119,\n 101,\n 51,\n 108,\n 82,\n 89,\n 73,\n 49,\n 118,\n 111,\n 55,\n 102,\n 68,\n 107,\n 75,\n 54,\n 82,\n 89,\n 82,\n 49,\n 75,\n 72,\n 104,\n 121,\n 69,\n 68,\n 82,\n 119,\n 109,\n 78,\n 85,\n 120,\n 70,\n 57,\n 84,\n 10,\n 109,\n 65,\n 51,\n 85,\n 116,\n 53,\n 98,\n 78,\n 87,\n 112,\n 118,\n 103,\n 101,\n 47,\n 98,\n 112,\n 77,\n 102,\n 80,\n 75,\n 99,\n 81,\n 102,\n 115,\n 113,\n 82,\n 121,\n 79,\n 69,\n 81,\n 70,\n 120,\n 111,\n 72,\n 68,\n 107,\n 79,\n 119,\n 100,\n 103,\n 121,\n 119,\n 49,\n 121,\n 53,\n 103,\n 106,\n 101,\n 72,\n 86,\n 114,\n 83,\n 117,\n 74,\n 47,\n 66,\n 101,\n 43,\n 56,\n 54,\n 112,\n 75,\n 74,\n 53,\n 10,\n 101,\n 49,\n 55,\n 77,\n 121,\n 47,\n 120,\n 87,\n 107,\n 121,\n 114,\n 109,\n 83,\n 83,\n 78,\n 108,\n 50,\n 84,\n 113,\n 83,\n 110,\n 84,\n 83,\n 116,\n 65,\n 76,\n 73,\n 67,\n 10,\n 45,\n 45,\n 45,\n 45,\n 45,\n 69,\n 78,\n 68,\n 32,\n 80,\n 82,\n 73,\n 86,\n 65,\n 84,\n 69,\n 32,\n 75,\n 69,\n 89,\n 45,\n 45,\n 45,\n 45,\n 45,\n 10,\n ],\n },\n ],\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.144258216Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Storage manager processing zpool: ZpoolInfo {\n name: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n size: 10200547328,\n allocated: 231936,\n free: 10200315392,\n health: Online,\n}","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.198333674Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Upserting synthetic disk for: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.198379145Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.198468098Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Storage manager processing zpool: ZpoolInfo {\n name: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n size: 10200547328,\n allocated: 236544,\n free: 10200310784,\n health: Online,\n}","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.261774165Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Upserting synthetic disk for: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.261810936Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.261908998Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.261915179Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.261919769Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Storage manager processing zpool: ZpoolInfo {\n name: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n size: 10200547328,\n allocated: 235008,\n free: 10200312320,\n health: Online,\n}","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.32316324Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.323266973Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.323289154Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.323330695Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.323343865Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.323356566Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.323363326Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.326436589Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Skipping rack secret creation (only one sled present)","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.326442139Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.326525491Z","hostname":"sock","pid":1097,"component":"BootstrapAgentRssHandler","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"SledAgentRequest { id: 6f0f865b-7d3c-40ef-a8b9-18c66c253d85, rack_id: 327b9e22-d506-4e4b-8546-82149102718d, ntp_servers: [\"ntp.eng.oxide.computer\"], dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }"}
{"msg":"No SP available; proceeding without sprockets auth","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.326679105Z","hostname":"sock","pid":1097,"BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.326703476Z","hostname":"sock","pid":1097,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:41388","component":"sprockets (BootstrapAgent)"}
{"msg":"No SP available; proceeding without sprockets auth","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.326749557Z","hostname":"sock","pid":1097,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:41388","component":"sprockets (BootstrapAgent)"}
{"msg":"Loading Sled Agent: SledAgentRequest { id: 6f0f865b-7d3c-40ef-a8b9-18c66c253d85, rack_id: 327b9e22-d506-4e4b-8546-82149102718d, ntp_servers: [\"ntp.eng.oxide.computer\"], dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.326827529Z","hostname":"sock","pid":1097,"component":"BootstrapAgent"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.32686943Z","hostname":"sock","pid":1097}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.326895691Z","hostname":"sock","pid":1097,"component":"DnsResolver","component":"NexusClient","addresses":"[[fd00:1122:3344:1::1]:53]"}
{"msg":"created sled agent","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.327083846Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.851425325Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.851796155Z","hostname":"sock","pid":1097,"component":"HardwareManager"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"da005c867f5e4d1daa7f756de235d90e\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.854286502Z","hostname":"sock","pid":1097,"component":"HardwareManager"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:15.854293772Z","hostname":"sock","pid":1097,"component":"HardwareManager"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.854313783Z","hostname":"sock","pid":1097,"component":"DnsResolver","component":"ServiceManager","component":"BootstrapAgent","addresses":"[[fd00:1122:3344:1::1]:53]"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.854561399Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.854885448Z","hostname":"sock","pid":1097,"component":"HardwareManager"}
{"msg":"Performing full hardware scan","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.854892938Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.854897048Z","hostname":"sock","pid":1097,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.854903028Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.854925819Z","hostname":"sock","pid":1097,"component":"BootstrapAgent"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.855434083Z","hostname":"sock","pid":1097,"DdmAdminClient":"[::1]:8000","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.855452343Z","hostname":"sock","pid":1097,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:41388","component":"sprockets (BootstrapAgent)"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.855527985Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","error":"proto error: io error: No route to host (os error 148)"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.855558396Z","hostname":"sock","pid":1097,"component":"BootstrapAgentRssHandler","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.855565546Z","hostname":"sock","pid":1097,"component":"BootstrapAgentRssHandler","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.855957037Z","hostname":"sock","pid":1097,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","remote_addr":"[fd00:1122:3344:101::1]:62420"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.856108931Z","hostname":"sock","pid":1097,"uri":"/zpools","method":"GET","req_id":"e76b5f7b-e063-4008-b99f-52212f456801","remote_addr":"[fd00:1122:3344:101::1]:62420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.856480791Z","hostname":"sock","pid":1097,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","remote_addr":"[fd00:1122:3344:101::1]:35489"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.856537452Z","hostname":"sock","pid":1097,"uri":"/sled-role","method":"GET","req_id":"bda27c86-3b47-4329-b8d6-9f5b9303265c","remote_addr":"[fd00:1122:3344:101::1]:35489","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"200"}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.857960191Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"sending service requests...","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.858040293Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.858070794Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.858186447Z","hostname":"sock","pid":1097,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","remote_addr":"[fd00:1122:3344:101::1]:45525"}
{"msg":"Ensuring service zone is initialized: InternalDns","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.858436244Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone internal_dns does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.858440844Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.873611943Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_internal_dns","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.890874849Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:15.915146943Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:16.031705305Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:16.133249453Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:16.278249213Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:16.553283039Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:16.834230075Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:17.049621422Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:17.443884904Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:17.516848871Z","hostname":"sock","pid":1097,"zone":"oxz_internal_dns","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:17.785866675Z","hostname":"sock","pid":1097,"zone":"oxz_switch","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:18.019264798Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:18.528081948Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:19.061945263Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:19.566139928Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:20.452740704Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:21.829038135Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"Ensuring address fd00:1122:3344:1::1 exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:22.091572424Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:22.091606585Z","hostname":"sock","pid":1097,"zone":"oxz_internal_dns","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address fd00:1122:3344:1::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:22.63065284Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"GZ addresses: [\n fd00:1122:3344:1::2,\n]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:22.630693581Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring GZ address fd00:1122:3344:1::2 exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:22.630699091Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.248337965Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.349797171Z","hostname":"sock","pid":1097,"DdmAdminClient":"[::1]:8000","component":"ServiceManager","component":"BootstrapAgent","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.350053978Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.354034205Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"0e930f8a-05eb-4d27-9e91-0c569122e936","remote_addr":"[fd00:1122:3344:101::1]:45525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.354420475Z","hostname":"sock","pid":1097,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS"}
{"msg":"failed to write DNS configuration (will retry in 87.004637ms)","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:23.354764925Z","hostname":"sock","pid":1097,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.445027908Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"sending service requests...","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.445237454Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.445264685Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.445392998Z","hostname":"sock","pid":1097,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","remote_addr":"[fd00:1122:3344:101::1]:57093"}
{"msg":"Ensuring service zone is initialized: Ntp","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.445917352Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone ntp does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.445924883Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.454602227Z","hostname":"sock","pid":1097,"component":"PortManager","port":"Port { inner: PortInner { name: \"opte0\", _ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 213, 81]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte0\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:23.454651448Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent","service":"ServiceZoneService { id: 0d607e9a-f782-4173-b84a-04929f7be674, details: BoundaryNtp { ntp_servers: [\"ntp.eng.oxide.computer\"], dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, kind: Service { id: 0d607e9a-f782-4173-b84a-04929f7be674 }, name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 192.168.1.22, first_port: 0, last_port: 16383 } } }"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:30.322134133Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:30.652989674Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:30.653019005Z","hostname":"sock","pid":1097,"zone":"oxz_switch","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe88:e715","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:31.194167816Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"GZ addresses: []","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:31.215027029Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Failed to initialize switch: Failed to do 'Adding Route' by running command in zone: Error running command in zone 'oxz_switch': Command [/usr/sbin/zlogin oxz_switch /usr/sbin/route add -inet6 default -inet6 fd00:1122:3344:101::1] executed and failed with status: exit status: 128 stdout: add net default: gateway fd00:1122:3344:101::1: Network is unreachable\n stderr: ","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:31.235714647Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Enabling switch zone (already underway)","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:31.235755068Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_switch in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:32.258468204Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:32.258508155Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Failed to initialize switch: Failed to install zone 'oxz_switch' from '/opt/oxide/switch.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_switch': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_switch': uninstall operation is invalid for shutting_down zones.","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:32.27539108Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:32.635083459Z","hostname":"sock","pid":1097,"zone":"oxz_switch","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:32.63510631Z","hostname":"sock","pid":1097,"zone":"oxz_switch","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:33.298016264Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:33.315341502Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:35.665509162Z","hostname":"sock","pid":1097,"zone":"oxz_switch","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to create NAT entry for service","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:38.456386075Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent","service":"ServiceZoneService { id: 0d607e9a-f782-4173-b84a-04929f7be674, details: BoundaryNtp { ntp_servers: [\"ntp.eng.oxide.computer\"], dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, kind: Service { id: 0d607e9a-f782-4173-b84a-04929f7be674 }, name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 192.168.1.22, first_port: 0, last_port: 16383 } } }","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12224/nat/ipv4/192.168.1.22/0/16383): operation timed out"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:38.552439395Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent","service":"ServiceZoneService { id: 0d607e9a-f782-4173-b84a-04929f7be674, details: BoundaryNtp { ntp_servers: [\"ntp.eng.oxide.computer\"], dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, kind: Service { id: 0d607e9a-f782-4173-b84a-04929f7be674 }, name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 192.168.1.22, first_port: 0, last_port: 16383 } } }"}
{"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:48.489251641Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:48.489288402Z","hostname":"sock","pid":1097,"zone":"oxz_switch","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Forwarding bootstrap traffic via oxBootstrap2 to fe80::8:20ff:fe88:e715","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:49.030714051Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:49.052277022Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:49.052307993Z","hostname":"sock","pid":1097,"zone":"oxz_switch","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:49.591752229Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"GZ addresses: []","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:49.5917819Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:49.745001541Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:50.160165365Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:50.414448062Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.500509363Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"34.645970794s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.501483639Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"35.178126393s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.501703635Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"35.178332409s"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.501829738Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"35.362171126s"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.501945811Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"35.178657628s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.502075025Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"35.178746541s"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.502213278Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"35.362504655s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.502340412Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"35.362827154s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.502442555Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"35.179203122s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.502561618Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"35.362865465s"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:50.502680301Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"35.179313705s"}
{"msg":"Setting up Maghemite service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:50.703542977Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:53.083329415Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"failed to create NAT entry for service","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:18:53.554252044Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent","service":"ServiceZoneService { id: 0d607e9a-f782-4173-b84a-04929f7be674, details: BoundaryNtp { ntp_servers: [\"ntp.eng.oxide.computer\"], dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, kind: Service { id: 0d607e9a-f782-4173-b84a-04929f7be674 }, name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 192.168.1.22, first_port: 0, last_port: 16383 } } }","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12224/nat/ipv4/192.168.1.22/0/16383): operation timed out"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:53.724314249Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent","service":"ServiceZoneService { id: 0d607e9a-f782-4173-b84a-04929f7be674, details: BoundaryNtp { ntp_servers: [\"ntp.eng.oxide.computer\"], dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, kind: Service { id: 0d607e9a-f782-4173-b84a-04929f7be674 }, name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 192.168.1.22, first_port: 0, last_port: 16383 } } }"}
{"msg":"Configuring new Omicron zone: oxz_ntp","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:53.741800131Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_ntp","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:53.758661785Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:54.553706013Z","hostname":"sock","pid":1097,"zone":"oxz_ntp","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address fd00:1122:3344:101::c exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:59.338187402Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::c, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:59.338242303Z","hostname":"sock","pid":1097,"zone":"oxz_ntp","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address fd00:1122:3344:101::c exists - OK","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:59.87993973Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"GZ addresses: []","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:18:59.879978601Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Set up NTP service boundary=true, Servers=[\"ntp.eng.oxide.computer\"]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:00.036951333Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:00.090087465Z","hostname":"sock","pid":1097,"zone":"oxz_ntp","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.391808141Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"d2f7a1dc-3121-4848-8a61-d2d7169e66e2","remote_addr":"[fd00:1122:3344:101::1]:57093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.392116169Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.392186262Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.392343926Z","hostname":"sock","pid":1097,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","remote_addr":"[fd00:1122:3344:101::1]:46381"}
{"msg":"Configured to skip timesync checks","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.39250795Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Setting boot time to 1684959545.392503s","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.39251329Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Updated /zone/oxz_internal_dns/root/var/adm/utmpx","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.397810133Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Updated /zone/oxz_internal_dns/root/var/adm/wtmpx","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.402721735Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Updated /zone/oxz_ntp/root/var/adm/utmpx","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.407516895Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Updated /zone/oxz_ntp/root/var/adm/wtmpx","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.412821648Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Updated /var/adm/utmpx","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.418153012Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Updated /var/adm/wtmpx","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.423170677Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.423226279Z","hostname":"sock","pid":1097,"uri":"/timesync","method":"GET","req_id":"04ab5fa2-8267-4d5a-aa70-02a09bdc2928","remote_addr":"[fd00:1122:3344:101::1]:46381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, skew: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.423570668Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"sending dataset requests...","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.423676511Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.423686091Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.423839405Z","hostname":"sock","pid":1097,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","remote_addr":"[fd00:1122:3344:101::1]:50743"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.424001209Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.466627929Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.46667058Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.481763527Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:05.499126565Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:06.579736543Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"51.256340556s"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:06.624043347Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"51.300691182s"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:06.667297624Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"51.34400231s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:06.757519867Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"51.434268274s"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:07.312868341Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"52.173191068s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:07.593153899Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"52.269776073s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:07.720383149Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"52.580862341s"}
{"msg":"Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:07.803410558Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:07.803566823Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:07.884945777Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"52.561606582s"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:08.084701923Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"52.230170914s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:08.109894962Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"52.970184438s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:08.180050354Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"53.04035001s"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:09.397396008Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.246592124Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"983f90b5-3d8e-404a-b42f-6355396d08d8","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService5] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService5 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:12.249608735Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"983f90b5-3d8e-404a-b42f-6355396d08d8\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"983f90b5-3d8e-404a-b42f-6355396d08d8\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.311679749Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.312006408Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.329075818Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.329100369Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.342583112Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.342601863Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.358953834Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"3f7e744e-7a41-401c-81d7-ce12021dcc70","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:12.361645206Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f7e744e-7a41-401c-81d7-ce12021dcc70\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3f7e744e-7a41-401c-81d7-ce12021dcc70\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.423792812Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.424053409Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.440244266Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.440281436Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.455135557Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.455167868Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.472543697Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"e8c16944-9bee-4541-96cd-dac1d532fc35","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:12.475242939Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e8c16944-9bee-4541-96cd-dac1d532fc35\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e8c16944-9bee-4541-96cd-dac1d532fc35\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.582975304Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.583217171Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.59876198Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.598792161Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.613411675Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.613434285Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.630718382Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"b5c368b0-2204-4c23-89dd-63efbaa4e966","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:12.63362489Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b5c368b0-2204-4c23-89dd-63efbaa4e966\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b5c368b0-2204-4c23-89dd-63efbaa4e966\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.806669336Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.806887432Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.822844972Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.822865213Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.838426303Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.838451983Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:12.856313295Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"9ad7f7ea-a3fe-46a3-a7f9-ce3a4fcdb140","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:12.858957826Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9ad7f7ea-a3fe-46a3-a7f9-ce3a4fcdb140\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9ad7f7ea-a3fe-46a3-a7f9-ce3a4fcdb140\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.008036305Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.008261341Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.024145749Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.02416358Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.03936961Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.03939259Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.056843841Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"d738d33a-7a89-453c-8874-a9ffdfbdcf2f","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:13.059276747Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d738d33a-7a89-453c-8874-a9ffdfbdcf2f\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d738d33a-7a89-453c-8874-a9ffdfbdcf2f\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.306871253Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.307092679Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.324457327Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.324488288Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Down","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.33975614Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.339786321Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.358639189Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"c8683b8c-4059-4ab7-b010-9dd98d668a7c","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:13.361353302Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c8683b8c-4059-4ab7-b010-9dd98d668a7c\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c8683b8c-4059-4ab7-b010-9dd98d668a7c\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.661327161Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.661598198Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.676711406Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.676728546Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.679521341Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.679532332Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.693773176Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:13.712416489Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:15.999343203Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:15.999550668Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:20.605076322Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"d753ef5c-4257-4527-a9d5-aab278b085ca","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService12] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService12 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:20.610833077Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d753ef5c-4257-4527-a9d5-aab278b085ca\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d753ef5c-4257-4527-a9d5-aab278b085ca\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:20.965793919Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:20.966146528Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:20.98142639Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:20.9814566Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:20.995222382Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:20.995243562Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.012275131Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"25f96b34-8bc5-4137-b373-6ede53407b36","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:21.014699817Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"25f96b34-8bc5-4137-b373-6ede53407b36\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"25f96b34-8bc5-4137-b373-6ede53407b36\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.400448518Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.400800748Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.416366928Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.416383608Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.430019206Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.430045436Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.447182429Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"a55827e6-3059-45e4-af2c-459ecf847877","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:21.449953563Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a55827e6-3059-45e4-af2c-459ecf847877\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a55827e6-3059-45e4-af2c-459ecf847877\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.922458454Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.922831384Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.940396418Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.940418659Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Incomplete","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.955755682Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:21.955778583Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:22.053652691Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:22.053701063Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:22.058388199Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"efa9b23d-5e5f-4fd5-a756-72997df5068e","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured\noxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:22.065558592Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"efa9b23d-5e5f-4fd5-a756-72997df5068e\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"efa9b23d-5e5f-4fd5-a756-72997df5068e\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:22.693643528Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:22.693938286Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:22.711196182Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:22.711224402Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:22.725546279Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:22.742755673Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:23.269456614Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"67.946084988s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:23.630117429Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"68.306866787s"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:23.756274701Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"68.432989288s"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:23.787403051Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"68.464060385s"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:23.869548475Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"68.729893643s"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:24.300800883Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"69.16108589s"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:24.399003391Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"68.544476953s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:24.726171404Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"69.402808558s"}
{"msg":"Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:24.92369311Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:24.923844844Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:25.428559232Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"70.105206978s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:25.680564818Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"70.540868174s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:26.044692396Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"70.905162927s"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:26.512536482Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:30.146283782Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"e49c3745-d302-4d40-a570-f3e0f920912e","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService16] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService16 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:30.149588371Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e49c3745-d302-4d40-a570-f3e0f920912e\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e49c3745-d302-4d40-a570-f3e0f920912e\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:30.892031981Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:30.892324078Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:30.912079471Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:30.912095722Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:30.929293106Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:30.929317826Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:30.948871673Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"757b8d78-5ff4-4707-83d8-7760cc5c2014","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/cockroachdb.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:30.951503304Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"757b8d78-5ff4-4707-83d8-7760cc5c2014\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"757b8d78-5ff4-4707-83d8-7760cc5c2014\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:31.641951081Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:31.641990142Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:32.081240106Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:32.081583095Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:32.099809596Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:32.099835597Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:32.115012057Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:32.13406511Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:34.326864767Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:34.327019601Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:38.93458451Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"1e80d05a-8fab-4272-a45d-b2a4dca03cc1","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService18] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService18 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:38.939934744Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1e80d05a-8fab-4272-a45d-b2a4dca03cc1\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1e80d05a-8fab-4272-a45d-b2a4dca03cc1\" }"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:40.167655508Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"84.844275811s"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:40.359194443Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:40.359215844Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:40.430261399Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:40.430615479Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:40.448122851Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:40.448143381Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:40.462029276Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:40.480094583Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:40.536503204Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"85.2131995s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:41.156913392Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"85.83366605s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:41.465222916Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"86.141858021s"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:41.512397208Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"86.372737676s"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:41.513554469Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"85.659031511s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:42.062826949Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"86.739487805s"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:42.096991181Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"86.773645205s"}
{"msg":"Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:42.612735267Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:42.612898482Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:42.786791791Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"87.647065977s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:43.040620824Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"87.900911861s"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:43.267191554Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:44.694647955Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"89.555105145s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:47.184400728Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"78854f79-3d01-49f9-a540-2b08a782a7b8","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService19] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService19 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:47.187141551Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78854f79-3d01-49f9-a540-2b08a782a7b8\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"78854f79-3d01-49f9-a540-2b08a782a7b8\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:48.586646198Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:48.586692629Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:49.0940795Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:49.094476401Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:49.113248137Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:49.113287958Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:49.127590634Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:49.146764561Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:51.268995095Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:51.269168479Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:55.648818624Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"469e4898-8975-4859-932a-5b97dd2430be","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService20] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService20 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:55.652625096Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"469e4898-8975-4859-932a-5b97dd2430be\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:19:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"469e4898-8975-4859-932a-5b97dd2430be\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:57.050897929Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:57.05092905Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, address: [fd00:1122:3344:101::6]:32221 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:57.451944603Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 484e7108-010b-4d4f-a3f4-9d5a07d52515, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: CockroachDb, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:57.452306973Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:57.469160817Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:57.469184598Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:57.484292185Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:57.503253536Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:57.636176851Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"102.312796184s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:58.065505376Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"102.742255004s"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:58.26885292Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"102.414322591s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:58.75912074Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"103.435747774s"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:59.622724116Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"104.483052923s"}
{"msg":"Profile for oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:59.623660981Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:19:59.623841936Z","hostname":"sock","pid":1097,"zone":"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:19:59.820281873Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"104.496982329s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:00.447979148Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"105.124641053s"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:00.597183091Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"105.273836886s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:01.193554961Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"106.053842698s"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:01.839706875Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"106.699978341s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:02.756952108Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"107.617421639s"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:03.337198013Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:03.819652352Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to format CRDB","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:03.997361524Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent","error":"RunCommandError { zone: \"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", err: CommandFailure(CommandFailureInfo { command: \"/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::6]:32221 --file /opt/oxide/cockroachdb/sql/dbwipe.sql\", status: ExitStatus(unix_wait_status(256)), stdout: \"\", stderr: \"ERROR: cannot dial server.\\nIs the server running?\\nIf the server is running, check --host client-side and --advertise server-side.\\n\\ndial tcp [fd00:1122:3344:101::6]:32221: connect: connection refused\\nFailed running \\\"sql\\\"\\n\" }) }"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:04.025557774Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to format CRDB","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:04.175460926Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent","error":"RunCommandError { zone: \"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", err: CommandFailure(CommandFailureInfo { command: \"/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::6]:32221 --file /opt/oxide/cockroachdb/sql/dbwipe.sql\", status: ExitStatus(unix_wait_status(256)), stdout: \"\", stderr: \"ERROR: cannot dial server.\\nIs the server running?\\nIf the server is running, check --host client-side and --advertise server-side.\\n\\ndial tcp [fd00:1122:3344:101::6]:32221: connect: connection refused\\nFailed running \\\"sql\\\"\\n\" }) }"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:04.307436545Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to format CRDB","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:04.459603358Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent","error":"RunCommandError { zone: \"oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", err: CommandFailure(CommandFailureInfo { command: \"/usr/sbin/zlogin oxz_cockroachdb_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::6]:32221 --file /opt/oxide/cockroachdb/sql/dbwipe.sql\", status: ExitStatus(unix_wait_status(256)), stdout: \"\", stderr: \"ERROR: cannot dial server.\\nIs the server running?\\nIf the server is running, check --host client-side and --advertise server-side.\\n\\ndial tcp [fd00:1122:3344:101::6]:32221: connect: connection refused\\nFailed running \\\"sql\\\"\\n\" }) }"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:04.621710799Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:12.782848517Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:12.783816913Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"1d988b97-e460-44a0-9513-3ce2a2609ff6","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 1f2e9de8-4e2a-4410-81a7-1f606e9e8c47, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Clickhouse, address: [fd00:1122:3344:101::7]:8123 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:12.784004898Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f2e9de8-4e2a-4410-81a7-1f606e9e8c47, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Clickhouse, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:12.784215254Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:12.841008935Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:12.841034476Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:12.841038686Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:12.841041946Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:12.858137247Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_clickhouse_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:12.876822571Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_clickhouse_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:15.01929241Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:15.019466824Z","hostname":"sock","pid":1097,"zone":"oxz_clickhouse_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:15.272810456Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"119.949430479s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:16.466204994Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"121.142953592s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:16.846067937Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"121.522691402s"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:18.328731055Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"123.189055223s"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:18.338968491Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"122.484444763s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:19.002220805Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"123.678884371s"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:19.199117144Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"123.875769079s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.199449324Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"f5ef5de3-f332-4922-a12b-e2096e0f1675","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.199607128Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.199775522Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.260710605Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.260751966Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.260756537Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.260760317Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.260763547Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.260767667Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.277704754Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:19.298093573Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:20.611469007Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:20.611599871Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:24.430700799Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.380370516Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"e3e037c0-ccd5-4e3f-ac98-5738456f42be","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService23] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService23 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:25.383053628Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e3e037c0-ccd5-4e3f-ac98-5738456f42be\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e3e037c0-ccd5-4e3f-ac98-5738456f42be\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.485993074Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.486491167Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.503612449Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.50364202Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.50364624Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.50365003Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.50366154Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.50366522Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.517990777Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.518013927Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.536600728Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"429047b6-e91e-4c18-b26d-7224d70ba5d2","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:25.539460566Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"429047b6-e91e-4c18-b26d-7224d70ba5d2\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"429047b6-e91e-4c18-b26d-7224d70ba5d2\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.686999184Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.687423675Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.70427231Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.70429975Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.704305921Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.704310191Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.704329001Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.704335362Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.719571282Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.719601293Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.737368072Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"00be8081-b8ba-4595-a6ee-985af134cb8d","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:25.740187658Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"00be8081-b8ba-4595-a6ee-985af134cb8d\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"00be8081-b8ba-4595-a6ee-985af134cb8d\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.874625443Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.874919151Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.891634122Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.891663743Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.891671733Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.891676013Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.891680163Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.891690323Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.907735226Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.907760387Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:25.925286289Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"ef21ae8c-10f4-476c-8546-3f2b8d096803","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:25.927828688Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ef21ae8c-10f4-476c-8546-3f2b8d096803\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ef21ae8c-10f4-476c-8546-3f2b8d096803\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.063256559Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.063565137Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.080224777Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.080269778Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.080274938Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.080280128Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.080284988Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.080299799Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.096442014Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.096459884Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.115857587Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"67350fbc-b142-4761-9304-b54df2394bdf","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:26.118275792Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"67350fbc-b142-4761-9304-b54df2394bdf\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"67350fbc-b142-4761-9304-b54df2394bdf\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.328206983Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.328580533Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.346882037Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.346906937Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.346912558Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.346918098Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.346923378Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.346934768Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.362680113Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.362700823Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.382146058Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"8e4eaaaa-cb70-4eb4-84e9-09c5776dbc22","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:26.38521485Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8e4eaaaa-cb70-4eb4-84e9-09c5776dbc22\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8e4eaaaa-cb70-4eb4-84e9-09c5776dbc22\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.589939541Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.590334452Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.609298113Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.609325633Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.609330504Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.609334864Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.609338984Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.609352154Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Incomplete","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.62401459Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.624046901Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.786332957Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.786398428Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:26.791223029Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"3c726e74-c9eb-4da6-9e55-5d6e1466bf5b","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured\noxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:26.798068413Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3c726e74-c9eb-4da6-9e55-5d6e1466bf5b\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3c726e74-c9eb-4da6-9e55-5d6e1466bf5b\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:27.10989641Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:27.11023471Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:27.128727208Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:27.128756709Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:27.12876773Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:27.128772399Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:27.12877652Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:27.1287806Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:27.144595416Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:27.163587038Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:27.846617886Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"132.523307332s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:27.846891143Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"132.70718534s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:27.847166261Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"132.707650363s"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:27.847301534Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"132.707592291s"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:28.568739537Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:28.568864711Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.357597514Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"b1ec6d87-32b4-4f42-9805-fc0fc2a3290a","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService30] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService30 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:33.360920403Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1ec6d87-32b4-4f42-9805-fc0fc2a3290a\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b1ec6d87-32b4-4f42-9805-fc0fc2a3290a\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.865431047Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.865817328Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.884343357Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.884369638Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.884373478Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.884377229Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.884380898Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.884384349Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.900156194Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.900189915Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:33.920446451Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"72eee2a2-9b37-494d-bca3-d69b040daecd","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:33.923218026Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"72eee2a2-9b37-494d-bca3-d69b040daecd\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"72eee2a2-9b37-494d-bca3-d69b040daecd\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.403054863Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.40329559Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.421358197Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.421385428Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.421393848Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.421398968Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.421404028Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.421409189Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Down","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.438357475Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.438395127Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.458382426Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"a671e7ea-37f4-4233-87f4-ffdf7f05397b","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:34.46114847Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a671e7ea-37f4-4233-87f4-ffdf7f05397b\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a671e7ea-37f4-4233-87f4-ffdf7f05397b\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.768726574Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:34.768748955Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:35.224911264Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:35.225261083Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:35.244079311Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:35.244106781Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:35.244111102Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:35.244116242Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:35.244120052Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:35.244133082Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:35.260290758Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:35.278272793Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:35.354587701Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"140.031334738s"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:36.382449216Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"141.05907192s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:36.441722645Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"141.11838635s"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:36.598692967Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:36.59880624Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:36.63848484Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"141.315104734s"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:37.345903034Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"142.206233212s"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:39.433301009Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"143.5787651s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:39.982706193Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"144.659338237s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:41.731831297Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"a5ebae1f-d555-4cd8-890e-233fa8eed1c1","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService33] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService33 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:41.734625992Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a5ebae1f-d555-4cd8-890e-233fa8eed1c1\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a5ebae1f-d555-4cd8-890e-233fa8eed1c1\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.483000851Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.483205537Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.501811298Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.501833879Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.501839179Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.50185001Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.50185422Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.501858Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.517479451Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.517507862Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:42.537320566Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"1420e94f-13d3-4ed1-9416-ab6502f01002","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:42.540146592Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1420e94f-13d3-4ed1-9416-ab6502f01002\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1420e94f-13d3-4ed1-9416-ab6502f01002\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.155804783Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.155835983Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.360862312Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.361146169Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.379383861Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.379416622Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.379421052Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.379424352Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.379427462Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.379430522Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.395480245Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:43.413683536Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:44.736463484Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:44.736619538Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:46.279779328Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:47.526770272Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"152.203457578s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:48.079707211Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"152.940183792s"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:48.14197444Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"153.002260546s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:48.45572919Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"153.316020756s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:49.866518341Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"53d745f5-e158-4e0b-906f-28b74ce2edb7","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService35] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService35 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:49.86983979Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53d745f5-e158-4e0b-906f-28b74ce2edb7\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53d745f5-e158-4e0b-906f-28b74ce2edb7\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.893237055Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.893504383Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.91194586Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.91196504Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.911970281Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.911982941Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.911986291Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.911989381Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Down","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.928528057Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.928563848Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:50.948978159Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"0a555630-00b2-4e73-a75e-cff4e2aef09b","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:50.951922848Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0a555630-00b2-4e73-a75e-cff4e2aef09b\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0a555630-00b2-4e73-a75e-cff4e2aef09b\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.294481644Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.294511935Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.750535121Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.75085006Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.770257363Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.770286484Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.770290804Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.770294654Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.770298444Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.770302165Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.786116851Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:51.804144907Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:53.117061238Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:53.117219412Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:55.359605386Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"160.219929323s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:55.428841083Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"160.10559611s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:55.618060235Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"160.29469327s"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:57.118366089Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"161.794983803s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:57.786507595Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"162.463157561s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:58.258544653Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"67f55cb5-77b0-4663-945e-3d782d5e7229","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService37] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService37 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:20:58.264643137Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"67f55cb5-77b0-4663-945e-3d782d5e7229\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:20:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"67f55cb5-77b0-4663-945e-3d782d5e7229\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.634434283Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.634789582Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.654649898Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.654675799Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.654680449Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.654684549Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.654693839Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.654698199Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Configured","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.673413104Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.673464855Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.680246458Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.680264369Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.683250889Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:20:59.702471948Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:01.016019185Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:01.016148599Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:01.282526141Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"165.427991843s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:02.295072414Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"166.971714438s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:06.228547326Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"13b7c842-c55b-4a55-afe2-7595dac226c1","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService38] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService38 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:06.234716963Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"13b7c842-c55b-4a55-afe2-7595dac226c1\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:21:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"13b7c842-c55b-4a55-afe2-7595dac226c1\" }"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:07.269391062Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"171.946088478s"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.658388361Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.658417721Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:07.71880399Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"172.579083486s"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.767858303Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.7681421Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.786545407Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.786567037Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.786571077Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.786574897Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.786578687Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.786582508Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.802179888Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:07.820704628Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:09.161607813Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:09.161729996Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:10.255510839Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:11.128961611Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"175.989419632s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:11.139261969Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"175.999557975s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:14.691633815Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"179.368251179s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:14.870594171Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"88e54b6f-c692-42e3-8241-5e206c427f9d","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService39] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService39 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:14.87390922Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"88e54b6f-c692-42e3-8241-5e206c427f9d\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:21:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"88e54b6f-c692-42e3-8241-5e206c427f9d\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:16.297127615Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:16.297153836Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:16.339444416Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"181.199784614s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:16.461726734Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"181.138479091s"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:17.316760029Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:17.316988555Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:17.335614837Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:17.335636188Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:17.335641618Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:17.335645518Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:17.335649288Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:17.335653148Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:17.350487908Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:17.369214853Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:18.68454253Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:18.684692964Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:18.763746986Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"183.440403951s"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:21.368048208Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"186.044660131s"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:23.349754233Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"188.026382466s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:24.236363529Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"b3feaedd-78c4-455e-b1f8-736e632d72c6","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService40] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService40 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:24.239624877Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b3feaedd-78c4-455e-b1f8-736e632d72c6\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:21:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b3feaedd-78c4-455e-b1f8-736e632d72c6\" }"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:25.258497721Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"189.403955822s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:25.676955274Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:25.676981705Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:26.651841961Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:26.652223231Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:26.671380778Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:26.671413559Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:26.671418169Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:26.671422489Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:26.671426679Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:26.671430909Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:26.686557767Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:26.705876468Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:26.865826141Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"191.542529258s"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:28.035843339Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:28.035998363Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:28.125938088Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"192.986221195s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:33.136177315Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"3019ba31-7af9-467a-a2fe-e5be0c5dd916","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService41] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService41 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:33.139271788Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3019ba31-7af9-467a-a2fe-e5be0c5dd916\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:21:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3019ba31-7af9-467a-a2fe-e5be0c5dd916\" }"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:33.28583759Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:33.666957847Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"198.527248493s"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:34.563704967Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:34.563733617Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:35.447554059Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"200.30802311s"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:36.476618156Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:36.476906784Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:36.496296527Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:36.496325698Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:36.496330838Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:36.496334858Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:36.496338728Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:36.496342508Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:36.512559285Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:36.532092382Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:37.148526604Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"201.825144448s"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:37.863675947Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:37.863821921Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:38.573411834Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"203.250147911s"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:40.915718753Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"205.77602906s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:43.238294258Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"f8c641c4-ddc8-4861-80fa-c10fc5dac0be","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService42] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService42 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:43.241292249Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f8c641c4-ddc8-4861-80fa-c10fc5dac0be\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:21:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f8c641c4-ddc8-4861-80fa-c10fc5dac0be\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:44.675983734Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:44.676019465Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:44.770685818Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"209.447349533s"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:44.903048557Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"209.57967938s"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:46.496701758Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"211.173343932s"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:46.820750816Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:46.820947441Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:46.839942083Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:46.839985254Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:46.839989864Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:46.839994254Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:46.839998365Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:46.840002835Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:46.855575695Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:46.875136442Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:47.694253619Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"212.554534105s"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:48.198835844Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:48.198965397Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:48.288580384Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"212.434046175s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:52.735802599Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"733c3e7a-4deb-4823-9071-e0e0234a9bda","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService43] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService43 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:52.739065757Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"733c3e7a-4deb-4823-9071-e0e0234a9bda\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:21:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"733c3e7a-4deb-4823-9071-e0e0234a9bda\" }"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:53.621261904Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"218.297960651s"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:54.161717647Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:54.161759248Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:56.534114147Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:57.961985579Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"222.638601533s"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:58.107023309Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:58.107207994Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:58.127550833Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:58.127584144Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:58.127587974Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:58.127591224Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:58.127594574Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:58.127598184Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:58.143418771Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:58.162454614Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:21:58.849745777Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"223.710017332s"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:59.299828042Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:21:59.299986886Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:00.368702983Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"225.045449001s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:02.335535297Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"227.196002868s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:04.012370281Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"de568a43-d251-4b8f-bbf7-677af2d233e6","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService44] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService44 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:04.015515836Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"de568a43-d251-4b8f-bbf7-677af2d233e6\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"de568a43-d251-4b8f-bbf7-677af2d233e6\" }"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:05.030686349Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"229.707310592s"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:05.456275165Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:05.456306556Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:05.792795979Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"230.469442914s"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:07.026668299Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"231.886996156s"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:08.516153421Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"233.376420267s"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:09.810243545Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:09.81043907Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:09.829950267Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:09.829974247Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:09.829979497Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:09.829983437Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:09.829986918Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:09.829990538Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:09.845768143Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:09.865060513Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:11.031873855Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:11.032040099Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:11.536867302Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"235.682336334s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:12.25701958Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"236.933662374s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:15.817339011Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"5d29134e-ae88-4df3-a123-5e3303a8fa1c","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService45] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService45 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:15.820418804Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5d29134e-ae88-4df3-a123-5e3303a8fa1c\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5d29134e-ae88-4df3-a123-5e3303a8fa1c\" }"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:16.689248711Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"241.365939417s"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:17.245011056Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:17.245051577Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:18.949882857Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:18.950269857Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:18.969431823Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:18.969467544Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:18.969471944Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:18.969476194Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:18.969480194Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:18.969484394Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:18.985294081Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:19.004037776Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:20.327950664Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:20.328098248Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:20.986622415Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"245.663249559s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:21.985096118Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"246.661840525s"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:23.00618907Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:24.479588379Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"249.339869535s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:24.72992895Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"249.590404281s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:24.985159021Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"94c52031-6d1d-4c03-90ea-9d169862154f","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService46] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService46 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:24.988010478Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"94c52031-6d1d-4c03-90ea-9d169862154f\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"94c52031-6d1d-4c03-90ea-9d169862154f\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:26.411630255Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:26.411667026Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:29.722654954Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"254.399256787s"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:30.994329202Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"255.854607849s"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:32.882451384Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:32.882706931Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:32.902237978Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:32.902275719Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:32.902280309Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:32.902284489Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:32.902288619Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:32.902292739Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:32.918235389Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:32.936743339Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:33.206514242Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"257.883177177s"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:34.07388503Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:34.074038774Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:34.121248817Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"258.981581544s"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:36.482884736Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"261.15949783s"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:37.451278098Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"262.127976794s"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:38.008099992Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"262.153568713s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:39.363137699Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"ca5218f9-741a-4d8f-8c01-6ffeb1ab3c40","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService47] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService47 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:39.366258833Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ca5218f9-741a-4d8f-8c01-6ffeb1ab3c40\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ca5218f9-741a-4d8f-8c01-6ffeb1ab3c40\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:40.80581562Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:40.805866171Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::8]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:44.727837743Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 713dda01-615d-4204-98da-83bb654cdcb8, zpool_id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:44.728193873Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:44.748295475Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:44.748326296Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:44.748329906Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:44.748333486Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:44.748337116Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:44.748340606Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:44.764259905Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:44.783235667Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:44.963164669Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"269.639790033s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:45.478314789Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"270.155064607s"}
{"msg":"Profile for oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:45.948137708Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:45.948292152Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.224356591Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"43c82571-b757-4b5c-afc0-e8587506dc30","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.224526786Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.224716661Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.270551227Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.270593268Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.270598698Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.270603128Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.270607328Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.270611678Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.270615549Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.270619439Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.287710699Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.306524977Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:50.643588036Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"Profile for oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:51.492582658Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:51.492687461Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:52.792461668Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"277.652747924s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:53.230831948Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"278.091298869s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.621161865Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"d2cf9b0d-f975-432e-9eb2-835d71a30898","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Command [/usr/sbin/zlogin oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b /usr/sbin/ipadm create-if -t oxControlService49] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService49 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:56.624166076Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2cf9b0d-f975-432e-9eb2-835d71a30898\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2cf9b0d-f975-432e-9eb2-835d71a30898\" }"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:56.718405097Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"281.578731814s"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.744453829Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.744771588Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.763565165Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.763619976Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.763628006Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.763634527Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.763640747Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.763647527Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.763653427Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.763659027Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.779538305Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.779569626Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.798892247Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"b1fbbd3a-ae1f-49f3-812c-cdef7d280e4a","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:56.801559949Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1fbbd3a-ae1f-49f3-812c-cdef7d280e4a\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b1fbbd3a-ae1f-49f3-812c-cdef7d280e4a\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.937024462Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.937300289Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.954082362Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.954108123Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.954112713Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.954116883Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.954121243Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.954125433Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.954129433Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.954133393Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.97073161Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.970764601Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:56.989675791Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"fc02c436-5209-43c0-a496-d0c3364f4be3","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:56.992486817Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fc02c436-5209-43c0-a496-d0c3364f4be3\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fc02c436-5209-43c0-a496-d0c3364f4be3\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.094827746Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.095146025Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.110742285Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.110777076Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.110781407Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.110785547Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.110789827Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.110794007Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.110798037Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.110807687Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.12462283Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.124652021Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.144853925Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"3c38a430-8167-4272-a6fd-0bf71f2118bd","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:57.147653901Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3c38a430-8167-4272-a6fd-0bf71f2118bd\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3c38a430-8167-4272-a6fd-0bf71f2118bd\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.332949287Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.333253465Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.348030654Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.348067495Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.348073575Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.348079565Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.348085785Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.348102076Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.348108156Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.348114056Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.363904922Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.363940243Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.383050658Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"40a6b44e-ac16-46b6-9514-ac70d672b7dc","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:57.385808763Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"40a6b44e-ac16-46b6-9514-ac70d672b7dc\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"40a6b44e-ac16-46b6-9514-ac70d672b7dc\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.592617529Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.592950198Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.609901965Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.609929976Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.609934106Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.609943226Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.609947336Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.609951356Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.609955247Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.609959227Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.623828261Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.623862442Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.642888325Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"9c71743c-6037-4d06-8a95-f763d9716d62","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:57.645721631Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9c71743c-6037-4d06-8a95-f763d9716d62\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9c71743c-6037-4d06-8a95-f763d9716d62\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.787088393Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.787499784Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.804229465Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.804269926Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.804274376Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.804278476Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.804282476Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.804286467Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.804290217Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.804294107Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state Incomplete","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.820957996Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:57.820989227Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.044106933Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.044137674Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.044633257Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"70b82277-195a-432e-b7e8-a40ba0fcb222","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 25\nstdout:\nINFO: omicron: uninstalling zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b...\nstderr:\nzoneadm: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b: could not reset state: Problem updating index file","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:22:58.051896883Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"70b82277-195a-432e-b7e8-a40ba0fcb222\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:22:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"70b82277-195a-432e-b7e8-a40ba0fcb222\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.357971896Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.358244513Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.377172624Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.377206105Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.377210525Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.377216345Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.377222205Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.377229545Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.377234065Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.377238366Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.393668129Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:58.413048001Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:59.622052531Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:22:59.622178824Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:00.214759022Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"284.891366935s"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:02.4205822Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"287.097267796s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.283197164Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"85b58404-56e7-4ec1-b987-a20f4f23a185","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Command [/usr/sbin/zlogin oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b /usr/sbin/ipadm create-if -t oxControlService56] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService56 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:04.286049501Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"85b58404-56e7-4ec1-b987-a20f4f23a185\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"85b58404-56e7-4ec1-b987-a20f4f23a185\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.571461217Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.571878978Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.589986566Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.590015357Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.590026937Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.590030817Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.590034337Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.590037838Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.590041198Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.590044578Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.605860004Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.605892735Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:04.626609574Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"88561607-5431-4536-9ac1-54945e6e4a1e","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:04.629359138Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"88561607-5431-4536-9ac1-54945e6e4a1e\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"88561607-5431-4536-9ac1-54945e6e4a1e\" }"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:04.682348127Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"289.542623353s"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.017445232Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.017681648Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.035348845Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.035386655Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.035390946Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.035395366Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.035399636Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.035403706Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.035407746Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.035411766Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.051564722Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.051587642Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.070150463Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"21a8048f-522f-4394-bea8-ead8834d64c6","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:05.07265191Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"21a8048f-522f-4394-bea8-ead8834d64c6\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"21a8048f-522f-4394-bea8-ead8834d64c6\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.475870443Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.476152831Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.495608185Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.495635086Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.495639756Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.495644116Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.495648436Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.495652697Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.495656597Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.495660627Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state Incomplete","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.513056056Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.513086456Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:05.645730393Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"289.791197965s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.719808051Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.719856072Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:05.724619231Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"123b7ac6-4fd0-457b-8836-2fa260933726","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b: No such zone configured\noxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b: No such zone configured","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:05.732167374Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"123b7ac6-4fd0-457b-8836-2fa260933726\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"123b7ac6-4fd0-457b-8836-2fa260933726\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.18003649Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.180243256Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.198018575Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.198051856Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.198055636Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.198059396Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.198062826Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.198066436Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.198069806Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.198073276Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.21414975Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:06.232130125Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:07.361440545Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:07.361573759Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:08.802545704Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"293.479198439s"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:11.804506078Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"296.481146383s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.096673976Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"88b73d9e-21b9-4a10-9780-b5f09193d9d8","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Command [/usr/sbin/zlogin oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b /usr/sbin/ipadm create-if -t oxControlService60] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService60 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:12.099747429Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"88b73d9e-21b9-4a10-9780-b5f09193d9d8\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"88b73d9e-21b9-4a10-9780-b5f09193d9d8\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.488510622Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.488888692Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.504456592Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.504488143Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.504493143Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.504498163Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.504503253Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.504508113Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.504512933Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.504517644Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.518764298Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.518793429Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:12.538024747Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"8a79b514-94ca-4b0f-8272-048450a3eab4","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:12.540819443Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8a79b514-94ca-4b0f-8272-048450a3eab4\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8a79b514-94ca-4b0f-8272-048450a3eab4\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.530991871Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.531023992Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.814294641Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.814731822Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.833445017Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.833466827Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.833471008Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.833475398Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.833479628Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.833483918Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.833488148Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.833492158Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.849299274Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:13.869228852Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:15.003369782Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:15.003469865Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:16.399368564Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"301.076112431s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:17.761133703Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"302.621594194s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:19.973593069Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"56d28c14-8717-4294-a173-1e7b178ac360","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Command [/usr/sbin/zlogin oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b /usr/sbin/ipadm create-if -t oxControlService62] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService62 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:19.976782125Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"56d28c14-8717-4294-a173-1e7b178ac360\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"56d28c14-8717-4294-a173-1e7b178ac360\" }"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:20.039969229Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"304.716591563s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:21.052356697Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"305.912565391s"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.114330638Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.114613476Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.136142347Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.136180957Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.136186288Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.136191008Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.136195488Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.136199728Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.136204048Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.136208478Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b in state Installed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.15483248Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.154859081Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.415637933Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.415688384Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.420325179Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"b1d3b5ce-42e6-41d6-9017-25fe2f930412","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b: No such zone configured\noxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b: No such zone configured","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:21.427320078Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1d3b5ce-42e6-41d6-9017-25fe2f930412\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b1d3b5ce-42e6-41d6-9017-25fe2f930412\" }"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:21.548382083Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, address: [fd00:1122:3344:101::9]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.502293704Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: b36b595f-0827-42f8-8f6d-70c9133799a1, zpool_id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.50254956Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.521702637Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.521734448Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.521739038Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.521742718Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.521746268Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.521749818Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.521753108Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.521761809Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.537690318Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:22.557721398Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:23.677442191Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:23.677585264Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:26.500917122Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"311.361239609s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.188150067Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"f45c6050-3c0f-4c31-84b4-4fdb954b2127","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.188309581Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.188508576Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.229222684Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.229267715Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.229271946Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.229275366Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.229278706Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.229282186Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.229285356Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.229288536Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.229291716Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.229294766Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.245804931Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:28.264566267Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:29.392422809Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:29.392540442Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:30.255122011Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"315.115393507s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.395195614Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"f67b19c2-d8e7-4684-afc7-7caf6c8aaedc","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService65] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService65 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:34.398103232Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f67b19c2-d8e7-4684-afc7-7caf6c8aaedc\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f67b19c2-d8e7-4684-afc7-7caf6c8aaedc\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.484778229Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.485243832Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.503023731Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.503055842Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.503064482Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.503070842Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.503077343Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.503083333Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.503089433Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.503096203Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.503117964Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.503123804Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.519224018Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.519243638Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.538991591Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"f80e7c1c-b51e-472f-b446-737bdc15f1f6","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:34.541450497Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f80e7c1c-b51e-472f-b446-737bdc15f1f6\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f80e7c1c-b51e-472f-b446-737bdc15f1f6\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.680904348Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.681148934Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.699327945Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.699361085Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.699365306Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.699369546Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.699378896Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.699383116Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.699387196Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.699391096Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.699395146Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.699398976Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.71548025Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.715515261Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.73439933Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"b2f1254d-616a-4e62-883d-bba989ac818f","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:34.73696666Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b2f1254d-616a-4e62-883d-bba989ac818f\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b2f1254d-616a-4e62-883d-bba989ac818f\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.938212336Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.938463203Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.954971378Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.955003539Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.955007899Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.955011929Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.955018229Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.955023689Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.955027589Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.955031129Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.955034269Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.95503754Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.971630407Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.971661397Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:34.990389252Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"fb9dce53-88c2-452f-b0d6-1cf1d4b005f9","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:34.993087035Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fb9dce53-88c2-452f-b0d6-1cf1d4b005f9\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fb9dce53-88c2-452f-b0d6-1cf1d4b005f9\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.233263141Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.233561749Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.251191105Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.251232496Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.251239036Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.251245096Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.251251106Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.251257256Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.251263457Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.251269207Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.251274947Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.251280657Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.267833643Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.267861434Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.286970399Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"fc073abf-62a4-41ba-90a4-000e8ef8d6ed","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:35.289527139Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fc073abf-62a4-41ba-90a4-000e8ef8d6ed\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fc073abf-62a4-41ba-90a4-000e8ef8d6ed\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.554600026Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.554845853Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.574917764Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.574956805Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.574962565Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.574966935Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.574971236Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.574975476Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.574979516Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.574983536Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.574987516Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.574991426Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Incomplete","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.592611001Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.592647752Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.82435212Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.824480483Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.829491959Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"e6d7805f-8a19-449f-9166-b40861b370c8","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured\noxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:35.836845197Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e6d7805f-8a19-449f-9166-b40861b370c8\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e6d7805f-8a19-449f-9166-b40861b370c8\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.965116125Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.965393143Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.983195912Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.983214533Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.983221903Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.983225523Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.983229073Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.983243114Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.983246784Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.983250044Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.983253484Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.983256714Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:35.994535078Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"320.671195473s"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:35.998589578Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:36.01573186Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:36.550955382Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"320.696419483s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:37.1379676Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"321.814591113s"}
{"msg":"Profile for oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:37.253118055Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:37.253279999Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:41.462391753Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"326.13908484s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:41.734990964Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"05639170-c7cf-40e6-8ee6-bc26bd4f27bb","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Command [/usr/sbin/zlogin oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 /usr/sbin/ipadm create-if -t oxControlService71] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService71 : Interface already exists","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:41.738039096Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"05639170-c7cf-40e6-8ee6-bc26bd4f27bb\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"05639170-c7cf-40e6-8ee6-bc26bd4f27bb\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.082747881Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.083012898Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.101587459Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.101619629Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.10162422Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.10162912Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.10163325Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.101661031Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.101665641Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.101669901Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.101673981Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.101678051Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.119136202Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.119164053Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.139660015Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"aba87740-e493-402d-b329-43ed705bdb0c","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:42.142503832Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aba87740-e493-402d-b329-43ed705bdb0c\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aba87740-e493-402d-b329-43ed705bdb0c\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.533088394Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.533410823Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.551583273Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.551626014Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.551630964Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.551635084Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.551639214Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.551643204Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.551647144Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.551651135Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.551655075Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.551658925Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.567399949Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.56742864Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:42.587146082Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"5f84dfdd-267e-4e0a-bff9-ed7a10c7822d","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:42.58969365Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5f84dfdd-267e-4e0a-bff9-ed7a10c7822d\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5f84dfdd-267e-4e0a-bff9-ed7a10c7822d\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.020791124Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.021116723Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.040470175Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.040494805Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.040501445Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.040505065Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.040508645Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.040512156Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.040515696Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.040519056Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.040522286Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.040525576Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 in state Incomplete","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.057449832Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.057475443Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.17009397Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.170209523Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.174865148Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"8b3d6b73-939e-4e99-b085-eb49e196505d","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03' from '/opt/oxide/crucible.tar.gz': Failed to execute zoneadm command 'Delete' for zone 'oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\noxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured\noxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03: No such zone configured","response_code":"500"}
{"msg":"failed to create filesystem","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:43.182467924Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8b3d6b73-939e-4e99-b085-eb49e196505d\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8b3d6b73-939e-4e99-b085-eb49e196505d\" }"}
{"msg":"creating new filesystem: DatasetEnsureBody { id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, address: [fd00:1122:3344:101::a]:32345 }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.825565034Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2681e087-4c51-42e2-a146-cb75c34bce74, zpool_id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, dataset_kind: Crucible, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.825843942Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CockroachDb","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.84466634Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone cockroachdb already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.844699261Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Clickhouse","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.84470383Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone clickhouse already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.844708461Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.844712841Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.844717371Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.844721661Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.844725691Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Crucible","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.844729781Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.844733871Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.861041051Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:43.879312754Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:44.671278288Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"329.53175481s"}
{"msg":"Profile for oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:44.993860216Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:44.99399731Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:47.877605954Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"332.554240648s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:48.870962569Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"333.547703036s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:49.305782353Z","hostname":"sock","pid":1097,"uri":"/filesystem","method":"PUT","req_id":"cad2ecb8-fecc-4583-9a1a-3ef5265eb872","remote_addr":"[fd00:1122:3344:101::1]:50743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"Finished setting up agents and datasets","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:49.306072351Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"sending service requests...","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:49.306326028Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:49.306360228Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:49.306507262Z","hostname":"sock","pid":1097,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","remote_addr":"[fd00:1122:3344:101::1]:51451"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:49.307036697Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:49.307042377Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:49.3260732Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:49.343960122Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_pantry:\n\n\n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:50.256683734Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:50.256808417Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:50.359589089Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"335.036208563s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:54.97097749Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"3d30412f-9f66-43c1-b7f1-b9126592e253","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService76] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService76 : Interface already exists","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:54.974477544Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3d30412f-9f66-43c1-b7f1-b9126592e253\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:23:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3d30412f-9f66-43c1-b7f1-b9126592e253\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:55.118134858Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: Oximeter","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:55.11895156Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone oximeter does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:55.11895723Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_oximeter","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:55.136264057Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_oximeter","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:55.154283603Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:23:55.807080545Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"340.667362661s"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:56.10803785Z","hostname":"sock","pid":1097,"zone":"oxz_oximeter","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:56.620331524Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:23:56.620368145Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address fd00:1122:3344:101::5 exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:00.721435656Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::5, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:00.721496057Z","hostname":"sock","pid":1097,"zone":"oxz_oximeter","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address fd00:1122:3344:101::5 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:01.259659208Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"GZ addresses: []","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:01.259707519Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Setting up oximeter service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:01.409453267Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:01.517612043Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:01.517645384Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:01.539224036Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:01.558148657Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:02.368776085Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"347.22904362s"}
{"msg":"Profile for oxz_crucible_pantry:\n\n\n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:02.45649124Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:02.456609143Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:04.054717974Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"348.915040811s"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:04.106751567Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:06.384173296Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"351.060789659s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:06.99701403Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"c9eac7c2-7a9c-4d2d-afc8-cbe5e3115c1e","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService78] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService78 : Interface already exists","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:07.000451062Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c9eac7c2-7a9c-4d2d-afc8-cbe5e3115c1e\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c9eac7c2-7a9c-4d2d-afc8-cbe5e3115c1e\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.163691154Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.164696042Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.164702282Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.181245718Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.181279698Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.201229456Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"6b1513b8-795b-472a-b039-58509d8ea63a","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:07.204466414Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6b1513b8-795b-472a-b039-58509d8ea63a\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6b1513b8-795b-472a-b039-58509d8ea63a\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.375782693Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: Oximeter","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.376629016Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone oximeter already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.376635976Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: ExternalDns","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.376639766Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone external_dns does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.376643317Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.384862888Z","hostname":"sock","pid":1097,"component":"PortManager","port":"Port { inner: PortInner { name: \"opte1\", _ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 255, 212, 24]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte1\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.384885679Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent","service":"ServiceZoneService { id: 8885de6b-6073-47f8-921c-c148b35c79d3, details: ExternalDns { http_address: [fd00:1122:3344:101::3]:5353, dns_address: 192.168.1.20:53, nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, kind: Service { id: 8885de6b-6073-47f8-921c-c148b35c79d3 }, name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 24])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }"}
{"msg":"Configuring new Omicron zone: oxz_external_dns","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.402140934Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_external_dns","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:07.420096268Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:08.580051175Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:08.580087746Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:08.65774871Z","hostname":"sock","pid":1097,"zone":"oxz_external_dns","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:12.334614763Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"357.011275079s"}
{"msg":"Ensuring address fd00:1122:3344:101::3 exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:13.255217956Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::3, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:13.255256417Z","hostname":"sock","pid":1097,"zone":"oxz_external_dns","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address fd00:1122:3344:101::3 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:13.79347751Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"GZ addresses: []","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:13.793505401Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Setting up external-dns service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:13.943024413Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:13.943051334Z","hostname":"sock","pid":1097,"zone":"oxz_external_dns","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:20.494365193Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone nexus does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:20.494402224Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:20.495479343Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","total duration":"364.640937704s","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::4]:12221/sled-agents/6f0f865b-7d3c-40ef-a8b9-18c66c253d85): operation timed out"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:20.495667758Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"365.356140139s"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:20.495964876Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"365.172618771s"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:20.507830116Z","hostname":"sock","pid":1097,"component":"PortManager","port":"Port { inner: PortInner { name: \"opte2\", _ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 178, 86]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:20.507860296Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent","service":"ServiceZoneService { id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, details: Nexus { internal_ip: fd00:1122:3344:101::4, external_ip: 192.168.1.21, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, kind: Service { id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }, name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 86])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }"}
{"msg":"Configuring new Omicron zone: oxz_nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:20.527902497Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:20.54654498Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:21.947407283Z","hostname":"sock","pid":1097,"zone":"oxz_nexus","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:23.13076198Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"367.807460257s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:24.99064863Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"369.850928146s"}
{"msg":"Ensuring address fd00:1122:3344:101::4 exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:27.043922655Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::4, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:27.043971366Z","hostname":"sock","pid":1097,"zone":"oxz_nexus","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address fd00:1122:3344:101::4 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:27.583189946Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"GZ addresses: []","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:27.583227777Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:27.73389252Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:27.733924131Z","hostname":"sock","pid":1097,"zone":"oxz_nexus","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:34.177848744Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"379.038096629s"}
{"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:34.178851131Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"379.039142468s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:34.179340534Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"378.856082471s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:34.17955831Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"378.856194575s"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, variant: U2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:34.179727295Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"378.85639882s"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:34.180529966Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"378.85716464s"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:34.20326673Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:34.20327492Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about Add { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, variant: M2 }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:34.216161307Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"379.076640139s"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:34.221803509Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:34.240208726Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_pantry:\n\n\n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:35.374733467Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:35.374890061Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:40.670138003Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"385.346777768s"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:40.76678513Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"b7b07d2c-3acf-4819-8239-5885ed879215","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService82] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService82 : Interface already exists","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:40.769937964Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b7b07d2c-3acf-4819-8239-5885ed879215\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b7b07d2c-3acf-4819-8239-5885ed879215\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.010967353Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.011756935Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.011765765Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.029694748Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.029724499Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.051394573Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"97a970c1-27bd-4c1f-b5a3-3ba2716629b1","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:41.054385404Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"97a970c1-27bd-4c1f-b5a3-3ba2716629b1\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"97a970c1-27bd-4c1f-b5a3-3ba2716629b1\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.324656012Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: Nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.325682079Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone nexus already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.32568993Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.32569368Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.32569725Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.342506313Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.342527754Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.364950858Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"2a269637-c724-4cf7-a458-34fe9a41509c","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:41.367815756Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2a269637-c724-4cf7-a458-34fe9a41509c\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2a269637-c724-4cf7-a458-34fe9a41509c\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.678171654Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: Oximeter","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.679213312Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone oximeter already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.679219173Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.679222933Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone nexus already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.679226443Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: ExternalDns","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.679232443Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone external_dns already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.679236013Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.679241143Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.679244663Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.696457867Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.696482488Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.718410099Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"dcf66fbc-a745-4f7c-a041-c6c695fefc6f","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:41.72140637Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dcf66fbc-a745-4f7c-a041-c6c695fefc6f\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dcf66fbc-a745-4f7c-a041-c6c695fefc6f\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.968016829Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.968851091Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.968859111Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_pantry in state Incomplete","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.987513094Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:41.987533575Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.181373992Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.181405213Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.18205846Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"d6a73482-81b1-4252-b856-c8457a1cc77a","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 25\nstdout:\nINFO: omicron: uninstalling zone oxz_crucible_pantry...\nstderr:\nzoneadm: oxz_crucible_pantry: could not reset state: Problem updating index file","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:42.185149904Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d6a73482-81b1-4252-b856-c8457a1cc77a\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d6a73482-81b1-4252-b856-c8457a1cc77a\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.679096873Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: Nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.679864564Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone nexus already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.679869274Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Oximeter","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.679872514Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone oximeter already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.679875564Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.679878644Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.679881694Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.698694622Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:42.71717157Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_pantry:\n\n\n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:43.796455362Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:43.796598265Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:47.221455623Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:47.227716122Z","hostname":"sock","pid":1097,"sled_id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85","component":"SledAgent","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"da005c867f5e4d1daa7f756de235d90e\" }","id":"6f0f865b-7d3c-40ef-a8b9-18c66c253d85"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:48.42327867Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"1b310639-e15e-4a1a-852d-502a0e2db746","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService87] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService87 : Interface already exists","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:48.42699554Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1b310639-e15e-4a1a-852d-502a0e2db746\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1b310639-e15e-4a1a-852d-502a0e2db746\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.014342557Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.015172709Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.01518061Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.033679348Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.033714309Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.056770021Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"510ffea7-c12b-4f4b-b791-7c4a706d6744","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:49.059563526Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"510ffea7-c12b-4f4b-b791-7c4a706d6744\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"510ffea7-c12b-4f4b-b791-7c4a706d6744\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.361491228Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.362270929Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.362277539Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.379698009Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.37973159Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.401375703Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"cd9fd7e2-ab79-4bbe-aa03-0cd4ab1c8d73","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:49.404469257Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cd9fd7e2-ab79-4bbe-aa03-0cd4ab1c8d73\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cd9fd7e2-ab79-4bbe-aa03-0cd4ab1c8d73\" }"}
{"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:49.714550048Z","hostname":"sock","pid":1097,"component":"StorageManager","component":"BootstrapAgent","total duration":"394.391170581s"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.759890761Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: Nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.760907788Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone nexus already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.760912848Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: ExternalDns","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.760928199Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone external_dns already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.760933629Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Oximeter","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.760937279Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone oximeter already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.760940909Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.760944249Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.760947619Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_pantry in state Incomplete","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.779408227Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.779444658Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.847634437Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.847658887Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:49.848611973Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"ab7d9d5e-8a8e-4db2-922d-a95f4063d3ac","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 25\nstdout:\nINFO: omicron: uninstalling zone oxz_crucible_pantry...\nstderr:\nzoneadm: oxz_crucible_pantry: could not reset state: Problem updating index file","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:49.848765677Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ab7d9d5e-8a8e-4db2-922d-a95f4063d3ac\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ab7d9d5e-8a8e-4db2-922d-a95f4063d3ac\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.827917949Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: Oximeter","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.828664269Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone oximeter already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.828676069Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: ExternalDns","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.82868261Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone external_dns already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.82868816Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.82869349Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone nexus already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.82869866Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.82870381Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.82871509Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.848461963Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:50.867471816Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_pantry:\n\n\n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:51.895473205Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:51.895709751Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:56.392261357Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"59f14e12-86cb-4474-b5fa-5c2e8e882786","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService91] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService91 : Interface already exists","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:56.396210833Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"59f14e12-86cb-4474-b5fa-5c2e8e882786\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"59f14e12-86cb-4474-b5fa-5c2e8e882786\" }"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:57.263694554Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: Nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:57.264973288Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone nexus already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:57.264982529Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:57.264994299Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:57.264998459Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"install_omicron_zone: Found zone: oxz_crucible_pantry in state ShuttingDown","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:57.28396765Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Invalid state; uninstalling and deleting zone oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:57.284008762Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:57.306318543Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"7b337b27-75fa-401d-85a8-dbdaace45e22","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to install zone 'oxz_crucible_pantry' from '/opt/oxide/crucible_pantry.tar.gz': Failed to execute zoneadm command 'Uninstall' for zone 'oxz_crucible_pantry': Failed to parse command output: exit code 1\nstdout:\n\nstderr:\nzoneadm: zone 'oxz_crucible_pantry': uninstall operation is invalid for shutting_down zones.","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:24:57.309398956Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7b337b27-75fa-401d-85a8-dbdaace45e22\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:24:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7b337b27-75fa-401d-85a8-dbdaace45e22\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:57.814447655Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:57.814498026Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:58.703583369Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: Nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:58.704782422Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone nexus already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:58.704789082Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Oximeter","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:58.704794572Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone oximeter already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:58.704798292Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:58.704807662Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:58.704811673Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:58.722866879Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:58.7406991Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_pantry:\n\n\n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:59.787326771Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:24:59.787554408Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:04.628673934Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"1e19cc4e-64cd-4c65-a238-debab66824a5","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService93] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService93 : Interface already exists","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:25:04.632323103Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1e19cc4e-64cd-4c65-a238-debab66824a5\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:25:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1e19cc4e-64cd-4c65-a238-debab66824a5\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:06.057746378Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:06.057783519Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:06.519897549Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:06.520915537Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:06.520933927Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:06.539337164Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:06.55812022Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_pantry:\n\n\n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:07.605492001Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:07.605679106Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:12.323900088Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"212bf972-0d60-44aa-aad0-8c8c0d4c4e5c","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","error_message_external":"Internal Server Error","error_message_internal":"Failed to boot zone: Zone booted, but running a command experienced an error: Error running command in zone 'oxz_crucible_pantry': Command [/usr/sbin/zlogin oxz_crucible_pantry /usr/sbin/ipadm create-if -t oxControlService94] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not create oxControlService94 : Interface already exists","response_code":"500"}
{"msg":"failed to initialize services","v":0,"name":"SledAgent","level":40,"time":"2023-05-24T20:25:12.329213711Z","hostname":"sock","pid":1097,"component":"RSS","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"212bf972-0d60-44aa-aad0-8c8c0d4c4e5c\", \"content-length\": \"124\", \"date\": \"Wed, 24 May 2023 20:25:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"212bf972-0d60-44aa-aad0-8c8c0d4c4e5c\" }"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:13.763099845Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:13.763156167Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"initializing sled services: [ServiceZoneRequest { addresses: [fd00:1122:3344:101::3], dataset: None, gz_addresses: [], id: 8885de6b-6073-47f8-921c-c148b35c79d3, services: [ServiceZoneService { details: ExternalDns { dns_address: \"192.168.1.20:53\", http_address: \"[fd00:1122:3344:101::3]:5353\", nic: NetworkInterface { id: 83094b40-989c-45be-b95d-776abe7ff314, ip: 172.30.0.5, kind: Service(8885de6b-6073-47f8-921c-c148b35c79d3), mac: MacAddr(\"A8:40:25:FF:D4:18\"), name: Name(\"external-dns-8885de6b-6073-47f8-921c-c148b35c79d3\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.0.0/24\")), vni: Vni(100) } }, id: 8885de6b-6073-47f8-921c-c148b35c79d3 }], zone_type: ExternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::4], dataset: None, gz_addresses: [], id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94, services: [ServiceZoneService { details: Nexus { external_ip: 192.168.1.21, internal_ip: fd00:1122:3344:101::4, nic: NetworkInterface { id: 9353204c-e53d-4f83-9841-f5c563379430, ip: 172.30.1.5, kind: Service(85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94), mac: MacAddr(\"A8:40:25:FF:B2:56\"), name: Name(\"nexus-85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.1.0/24\")), vni: Vni(100) } }, id: 85b4bfba-b5fe-4d7f-bcc6-05c3f8e88d94 }], zone_type: Nexus }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::5], dataset: None, gz_addresses: [], id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e, services: [ServiceZoneService { details: Oximeter, id: 88d3cd7d-e0bb-473e-800c-fbdb772cb12e }], zone_type: Oximeter }, ServiceZoneRequest { addresses: [fd00:1122:3344:1::1], dataset: None, gz_addresses: [fd00:1122:3344:1::2], id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd, services: [ServiceZoneService { details: InternalDns { dns_address: \"[fd00:1122:3344:1::1]:53\", http_address: \"[fd00:1122:3344:1::1]:5353\" }, id: 79d7fab0-92b4-4b68-90ff-caa4d38648dd }], zone_type: InternalDns }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::b], dataset: None, gz_addresses: [], id: 79c703a4-b1ad-4b79-9a57-544c96791c05, services: [ServiceZoneService { details: CruciblePantry, id: 79c703a4-b1ad-4b79-9a57-544c96791c05 }], zone_type: CruciblePantry }, ServiceZoneRequest { addresses: [fd00:1122:3344:101::c], dataset: None, gz_addresses: [], id: 0d607e9a-f782-4173-b84a-04929f7be674, services: [ServiceZoneService { details: BoundaryNtp { dns_servers: [\"1.1.1.1\", \"9.9.9.9\"], domain: None, nic: NetworkInterface { id: 8136acea-7679-4386-801f-0d7d2a942da1, ip: 172.30.2.5, kind: Service(0d607e9a-f782-4173-b84a-04929f7be674), mac: MacAddr(\"A8:40:25:FF:D5:51\"), name: Name(\"ntp-0d607e9a-f782-4173-b84a-04929f7be674\"), primary: true, slot: 0, subnet: V4(Ipv4Net(\"172.30.2.0/24\")), vni: Vni(100) }, ntp_servers: [\"ntp.eng.oxide.computer\"], snat_cfg: SourceNatConfig { first_port: 0, ip: 192.168.1.22, last_port: 16383 } }, id: 0d607e9a-f782-4173-b84a-04929f7be674 }], zone_type: Ntp }]","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:14.477149828Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Ensuring service zone is initialized: CruciblePantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:14.478022632Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone crucible_pantry does not yet exist","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:14.478031362Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:14.497391314Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:14.516563681Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Profile for oxz_crucible_pantry:\n\n\n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:15.57531845Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:15.575458223Z","hostname":"sock","pid":1097,"zone":"oxz_crucible_pantry","component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.854076863Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone nexus already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.854141585Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: ExternalDns","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.854152625Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone external_dns already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.854156805Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Ensuring service zone is initialized: Oximeter","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.854160706Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"Service zone oximeter already exists","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.854164596Z","hostname":"sock","pid":1097,"component":"ServiceManager","component":"BootstrapAgent"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.855689027Z","hostname":"sock","pid":1097,"uri":"/services","method":"PUT","req_id":"2f24cb02-efae-44fc-969b-34093ea0b823","remote_addr":"[fd00:1122:3344:101::1]:51451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","response_code":"204"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.855957754Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.85620317Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.856218081Z","hostname":"sock","pid":1097,"component":"DnsResolver","component":"RSS","addresses":"[[fd00:1122:3344:1::1]:53]"}
{"msg":"Nexus address: [fd00:1122:3344:101::4]:12221","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:19.857506496Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f5705cf0-3aed-4b20-8521-77aba3c42d56\", \"content-length\": \"174\", \"date\": \"Wed, 24 May 2023 20:25:19 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"f5705cf0-3aed-4b20-8521-77aba3c42d56\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:20.430275689Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e780198e-cdba-4bf2-b0ce-727824bf20d3\", \"content-length\": \"174\", \"date\": \"Wed, 24 May 2023 20:25:19 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"e780198e-cdba-4bf2-b0ce-727824bf20d3\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:20.615259117Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9203c3c2-7556-4c72-b666-3404fb1c7668\", \"content-length\": \"174\", \"date\": \"Wed, 24 May 2023 20:25:19 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"9203c3c2-7556-4c72-b666-3404fb1c7668\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:20.836225846Z","hostname":"sock","pid":1097,"component":"RSS"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb4f93b5-0f8a-405a-8acc-d53fed0a1f01\", \"content-length\": \"174\", \"date\": \"Wed, 24 May 2023 20:25:20 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"bb4f93b5-0f8a-405a-8acc-d53fed0a1f01\" }","v":0,"name":"SledAgent","level":30,"time":"2023-05-24T20:25:21.061465789Z","hostname":"sock","pid":1097,"component":"RSS"}