[ Aug 1 01:16:18 Enabled. ] [ Aug 1 01:16:18 Rereading configuration. ] [ Aug 1 01:16:19 Rereading configuration. ] [ Aug 1 01:16:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:21.064553874Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989} {"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:21.070106553Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:21.070134694Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:21.100298742Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:21.288819082Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.158011734Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:34022"} {"msg":"region 7333f72c-f690-467d-a376-9ba33de50f6b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.171493944Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.173595541Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions","method":"POST","req_id":"e6017bc6-96e3-4bcc-a285-aa2d04017c6f","remote_addr":"[fd00:1122:3344:101::b]:34022","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":8414,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.174200422Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7333f72c-f690-467d-a376-9ba33de50f6b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.185059006Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7333f72c-f690-467d-a376-9ba33de50f6b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.185091066Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"7333f72c-f690-467d-a376-9ba33de50f6b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7333f72c-f690-467d-a376-9ba33de50f6b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.252780663Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7333f72c-f690-467d-a376-9ba33de50f6b","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7333f72c-f690-467d-a376-9ba33de50f6b\" \"--data\" \"/data/regions/7333f72c-f690-467d-a376-9ba33de50f6b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.254677377Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7333f72c-f690-467d-a376-9ba33de50f6b","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.39236577Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7333f72c-f690-467d-a376-9ba33de50f6b","component":"worker"} {"msg":"region 7333f72c-f690-467d-a376-9ba33de50f6b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.392442911Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.392663705Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-7333f72c-f690-467d-a376-9ba33de50f6b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.393075963Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.401667896Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions","method":"POST","req_id":"0bd6ff17-3e2b-423a-b23f-cc55dbd5f6e0","remote_addr":"[fd00:1122:3344:101::b]:34022","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":76,"response_code":"200"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7333f72c-f690-467d-a376-9ba33de50f6b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.401952611Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.428017395Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7333f72c-f690-467d-a376-9ba33de50f6b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.435823154Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7333f72c-f690-467d-a376-9ba33de50f6b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.436516567Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.440713872Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.440782883Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.440825904Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.451372071Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.459197431Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.914974455Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7333f72c-f690-467d-a376-9ba33de50f6b","method":"GET","req_id":"1ba06a3d-5c1d-43f2-aa6b-161cf8b2041d","remote_addr":"[fd00:1122:3344:101::b]:34022","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":84,"response_code":"200"} {"msg":"path is \"/data/regions/7333f72c-f690-467d-a376-9ba33de50f6b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.91693997Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"dataset is oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7333f72c-f690-467d-a376-9ba33de50f6b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.926090003Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"region 7333f72c-f690-467d-a376-9ba33de50f6b state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.935235046Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.93546553Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.935544301Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7333f72c-f690-467d-a376-9ba33de50f6b","method":"DELETE","req_id":"c2d7abed-5063-4fb5-8f81-d9342773de9a","remote_addr":"[fd00:1122:3344:101::b]:34022","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":18659,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-7333f72c-f690-467d-a376-9ba33de50f6b (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.93660226Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.937641279Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7333f72c-f690-467d-a376-9ba33de50f6b","method":"GET","req_id":"801ccbed-2dca-4cff-9ab7-af6a316c535c","remote_addr":"[fd00:1122:3344:101::b]:34022","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":24,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.941120961Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7333f72c-f690-467d-a376-9ba33de50f6b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:58.950547679Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7333f72c-f690-467d-a376-9ba33de50f6b","component":"worker"} {"msg":"region 7333f72c-f690-467d-a376-9ba33de50f6b state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:59.014910816Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:16:59.087757464Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7333f72c-f690-467d-a376-9ba33de50f6b","method":"GET","req_id":"013e8a0f-099e-4d64-8c20-f527eae0cbdc","remote_addr":"[fd00:1122:3344:101::b]:34022","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":60,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.201635776Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59669"} {"msg":"region 7fe94aeb-b8c2-44ae-8848-80e99322feef state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.201740578Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.203970808Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions","method":"POST","req_id":"0135820a-596e-414b-9d24-ce044d55db18","remote_addr":"[fd00:1122:3344:101::a]:59669","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":2262,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.204004179Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.213204623Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.213245583Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"7fe94aeb-b8c2-44ae-8848-80e99322feef\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.264055009Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7fe94aeb-b8c2-44ae-8848-80e99322feef","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7fe94aeb-b8c2-44ae-8848-80e99322feef\" \"--data\" \"/data/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.26410873Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7fe94aeb-b8c2-44ae-8848-80e99322feef","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.332905045Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7fe94aeb-b8c2-44ae-8848-80e99322feef","component":"worker"} {"msg":"region 7fe94aeb-b8c2-44ae-8848-80e99322feef state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.332961186Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.333056068Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-7fe94aeb-b8c2-44ae-8848-80e99322feef","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.334454193Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7fe94aeb-b8c2-44ae-8848-80e99322feef","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.337992986Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.345456469Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7fe94aeb-b8c2-44ae-8848-80e99322feef","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.349012472Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.349378729Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.34945232Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.349505101Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.349559232Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.35337085Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.357963802Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:17:35.382259625Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions","method":"POST","req_id":"7d9db6f6-bda7-4617-840d-f49bda299e69","remote_addr":"[fd00:1122:3344:101::a]:59669","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:41.829197516Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56041"} {"msg":"region 764af782-c613-46be-8ba1-70a334e19fd8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:41.829343759Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:41.830288136Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:41.830298826Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions","method":"POST","req_id":"959c25eb-2305-42e8-8dd9-1240f238c4aa","remote_addr":"[fd00:1122:3344:101::a]:56041","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":994,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/764af782-c613-46be-8ba1-70a334e19fd8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:41.839900647Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/764af782-c613-46be-8ba1-70a334e19fd8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:41.839940648Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"764af782-c613-46be-8ba1-70a334e19fd8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/764af782-c613-46be-8ba1-70a334e19fd8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:41.895793473Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"764af782-c613-46be-8ba1-70a334e19fd8","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"764af782-c613-46be-8ba1-70a334e19fd8\" \"--data\" \"/data/regions/764af782-c613-46be-8ba1-70a334e19fd8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:41.895848314Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"764af782-c613-46be-8ba1-70a334e19fd8","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.014661012Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"764af782-c613-46be-8ba1-70a334e19fd8","component":"worker"} {"msg":"region 764af782-c613-46be-8ba1-70a334e19fd8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.014717703Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.014791924Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-764af782-c613-46be-8ba1-70a334e19fd8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.015747561Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-764af782-c613-46be-8ba1-70a334e19fd8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.024147161Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.028456028Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions","method":"POST","req_id":"33855d35-e41e-4fc0-88ea-01a9c475973f","remote_addr":"[fd00:1122:3344:101::a]:56041","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":647,"response_code":"200"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.041300227Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-764af782-c613-46be-8ba1-70a334e19fd8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.044368571Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/764af782-c613-46be-8ba1-70a334e19fd8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.044641706Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.044695807Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.044738548Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.044778169Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.049212698Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:42.055234995Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.127066148Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63243"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.12716314Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef","method":"GET","req_id":"83c6cb24-4c6b-495c-b255-f164ef82c9e0","remote_addr":"[fd00:1122:3344:101::a]:63243","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":55,"response_code":"200"} {"msg":"path is \"/data/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.128345911Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"dataset is oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.137627016Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"snapshot is oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef@88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.14569236Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef\", \"88052840-5a63-42fd-bb55-9316b8dcf787\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.145717461Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.157210866Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef/snapshots/88052840-5a63-42fd-bb55-9316b8dcf787","method":"GET","req_id":"e21b29c1-3510-4cd8-92f0-1dcc8ebeee3f","remote_addr":"[fd00:1122:3344:101::a]:63243","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":29780,"response_code":"200"} {"msg":"path is \"/data/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.15747448Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"dataset is oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.165162177Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"snapshot is oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef@88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.17207419Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef\", \"88052840-5a63-42fd-bb55-9316b8dcf787\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.172096861Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 7fe94aeb-b8c2-44ae-8848-80e99322feef-88052840-5a63-42fd-bb55-9316b8dcf787 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.183545285Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"applying SMF actions for region 7fe94aeb-b8c2-44ae-8848-80e99322feef running snapshot 88052840-5a63-42fd-bb55-9316b8dcf787 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.183733138Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.183768869Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef/snapshots/88052840-5a63-42fd-bb55-9316b8dcf787/run","method":"POST","req_id":"53cd03d2-b735-45c0-8208-73339df265f0","remote_addr":"[fd00:1122:3344:101::a]:63243","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":26341,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-7fe94aeb-b8c2-44ae-8848-80e99322feef-88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.186505228Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-7fe94aeb-b8c2-44ae-8848-80e99322feef-88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.189635503Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-7fe94aeb-b8c2-44ae-8848-80e99322feef-88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.201514195Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7fe94aeb-b8c2-44ae-8848-80e99322feef/.zfs/snapshot/88052840-5a63-42fd-bb55-9316b8dcf787","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.201728909Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.20177939Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.201823701Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.201862261Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.201913152Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.205828962Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.210083828Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"region 7fe94aeb-b8c2-44ae-8848-80e99322feef running snapshot 88052840-5a63-42fd-bb55-9316b8dcf787 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:43.210097048Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"region 7ffdd473-b1ef-4e23-9548-218c95ebb0eb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.092089733Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.092267256Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions","method":"POST","req_id":"83595c4f-b2be-4dce-9e8b-e4f90a5dbb0d","remote_addr":"[fd00:1122:3344:101::a]:56041","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":237,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.092299347Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.101171715Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.101202036Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"7ffdd473-b1ef-4e23-9548-218c95ebb0eb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.158726851Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7ffdd473-b1ef-4e23-9548-218c95ebb0eb","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7ffdd473-b1ef-4e23-9548-218c95ebb0eb\" \"--data\" \"/data/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.158775932Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7ffdd473-b1ef-4e23-9548-218c95ebb0eb","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.22824491Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7ffdd473-b1ef-4e23-9548-218c95ebb0eb","component":"worker"} {"msg":"region 7ffdd473-b1ef-4e23-9548-218c95ebb0eb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.228286551Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.228379222Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-7ffdd473-b1ef-4e23-9548-218c95ebb0eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.230850546Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7ffdd473-b1ef-4e23-9548-218c95ebb0eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.234262157Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.243535252Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7ffdd473-b1ef-4e23-9548-218c95ebb0eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.246926743Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.247252089Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.24730999Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.24736025Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.247400731Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.25123387Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.257392459Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:22:46.412276039Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions","method":"POST","req_id":"3f9bd97c-6166-4708-8d0e-f625ca067cf1","remote_addr":"[fd00:1122:3344:101::a]:56041","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":81,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.649449384Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb","method":"GET","req_id":"bf65fc28-f167-4f97-8933-a467ad25c45b","remote_addr":"[fd00:1122:3344:101::a]:56041","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":573,"response_code":"200"} {"msg":"path is \"/data/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.651650053Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"dataset is oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.662315183Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"region 7ffdd473-b1ef-4e23-9548-218c95ebb0eb state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.671297864Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.671489757Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.671557278Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb","method":"DELETE","req_id":"d18a6c85-741d-4d60-9f59-29e376f6a7c9","remote_addr":"[fd00:1122:3344:101::a]:56041","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19944,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-7ffdd473-b1ef-4e23-9548-218c95ebb0eb (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.673236398Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.674028442Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb","method":"GET","req_id":"1f4aac6a-6510-4b8e-aaa6-348157ee1d44","remote_addr":"[fd00:1122:3344:101::a]:56041","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":31,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.680279024Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_93aa4101-a141-482b-89ba-423a8d6fb4d2/crucible/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.690208241Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"region":"7ffdd473-b1ef-4e23-9548-218c95ebb0eb","component":"worker"} {"msg":"region 7ffdd473-b1ef-4e23-9548-218c95ebb0eb state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.750680788Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-01T01:23:10.808398477Z","hostname":"oxz_crucible_5279b1df-2efe-4d42-b519-f82a57f5f3f4","pid":22989,"uri":"/crucible/0/regions/7ffdd473-b1ef-4e23-9548-218c95ebb0eb","method":"GET","req_id":"212ed21b-71a4-42d6-bffd-3816c72757c8","remote_addr":"[fd00:1122:3344:101::a]:56041","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":72,"response_code":"200"}