[ Mar 1 23:17:33 Enabled. ] [ Mar 1 23:17:33 Rereading configuration. ] [ Mar 1 23:17:38 Rereading configuration. ] [ Mar 1 23:17:44 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:44.431168133Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789} {"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:44.434415023Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:44.434426813Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:44.548295492Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.472322221Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49218"} {"msg":"region b14d9f9b-dd00-4a5e-b35e-ca70c6463a96 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.474808441Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.475010646Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions","method":"POST","req_id":"24975531-46ec-4615-9326-bde7b34f396c","remote_addr":"[fd00:1122:3344:101::a]:49218","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":1276,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.475763365Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.484252503Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.484286914Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"b14d9f9b-dd00-4a5e-b35e-ca70c6463a96\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.531322666Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"region":"b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.668827134Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"region":"b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","component":"worker"} {"msg":"region b14d9f9b-dd00-4a5e-b35e-ca70c6463a96 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.668871015Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.668984958Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.66946703Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.675090198Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.687082992Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.691006068Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.691858089Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.692013553Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.692060134Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.692100245Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.697110867Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.702373916Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.747910352Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions","method":"POST","req_id":"72e67d64-6cf3-4420-8603-f0538c4218f6","remote_addr":"[fd00:1122:3344:101::a]:49218","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.300802912Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47746"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.30316799Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","method":"GET","req_id":"19a072a1-ba41-4f74-b92e-fd1b865d81ab","remote_addr":"[fd00:1122:3344:101::a]:47746","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":2339,"response_code":"200"} {"msg":"path is \"/data/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.303314304Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.312342135Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"snapshot is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96@8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.321410437Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96\", \"8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.321446528Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.332292104Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96/snapshots/8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","method":"GET","req_id":"17326508-c330-4db7-87a7-7a8d68dd040c","remote_addr":"[fd00:1122:3344:101::a]:47746","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":28989,"response_code":"200"} {"msg":"path is \"/data/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.332425417Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.340380122Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"snapshot is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96@8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.349216268Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96\", \"8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.349250149Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot b14d9f9b-dd00-4a5e-b35e-ca70c6463a96-8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.358115216Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"applying SMF actions for region b14d9f9b-dd00-4a5e-b35e-ca70c6463a96 running snapshot 8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.358733761Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.358758872Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96/snapshots/8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0/run","method":"POST","req_id":"289e06fb-807f-4a48-8d51-5c836598d913","remote_addr":"[fd00:1122:3344:101::a]:47746","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26361,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-b14d9f9b-dd00-4a5e-b35e-ca70c6463a96-8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.363084188Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-b14d9f9b-dd00-4a5e-b35e-ca70c6463a96-8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.457318606Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-b14d9f9b-dd00-4a5e-b35e-ca70c6463a96-8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.565011844Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b14d9f9b-dd00-4a5e-b35e-ca70c6463a96/.zfs/snapshot/8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.565247159Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.565327821Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.565390413Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.565446984Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.565507596Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.610515008Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.638872883Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"region b14d9f9b-dd00-4a5e-b35e-ca70c6463a96 running snapshot 8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.638907644Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:35.773373432Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59279"} {"msg":"region 7a19008c-9c9c-4036-aa51-5a4ee7396154 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:35.774142241Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:35.774300575Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions","method":"POST","req_id":"cf760395-35fb-48c6-b8c2-66e283df788c","remote_addr":"[fd00:1122:3344:101::a]:59279","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":909,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:35.774320565Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:35.78308142Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:35.783117471Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:35.963172132Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions","method":"POST","req_id":"eedd14b7-e64c-4912-ae2a-0f3a723e2fb7","remote_addr":"[fd00:1122:3344:101::a]:59279","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":78,"response_code":"200"} {"msg":"creating region Region { id: RegionId(\"7a19008c-9c9c-4036-aa51-5a4ee7396154\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:35.98923126Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"region":"7a19008c-9c9c-4036-aa51-5a4ee7396154","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.267816265Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"region":"7a19008c-9c9c-4036-aa51-5a4ee7396154","component":"worker"} {"msg":"region 7a19008c-9c9c-4036-aa51-5a4ee7396154 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.267873096Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.267963628Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-7a19008c-9c9c-4036-aa51-5a4ee7396154","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.268470631Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.28884606Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions","method":"POST","req_id":"3cdbe004-1dce-458a-a31f-daef17b8adcc","remote_addr":"[fd00:1122:3344:101::a]:59279","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":120,"response_code":"200"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7a19008c-9c9c-4036-aa51-5a4ee7396154","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.297571414Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.347421025Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7a19008c-9c9c-4036-aa51-5a4ee7396154","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.367905226Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.368337347Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.368393429Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.36844481Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.368491731Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.385009605Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:36.435021891Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:21.886998449Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63218"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:21.887150873Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154","method":"GET","req_id":"4c0c5f21-4220-4e25-8238-6a3367c07f44","remote_addr":"[fd00:1122:3344:101::a]:63218","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":101,"response_code":"200"} {"msg":"path is \"/data/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:21.887337698Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:21.937132758Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"region 7a19008c-9c9c-4036-aa51-5a4ee7396154 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:22.006481956Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:22.006670541Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:22.006705122Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154","method":"DELETE","req_id":"e9a63d19-0ba1-4b84-9bda-23896ca62607","remote_addr":"[fd00:1122:3344:101::a]:63218","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":119401,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:22.00742953Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154","method":"GET","req_id":"2ce2a2e2-bcfb-4015-95c0-3456fe823086","remote_addr":"[fd00:1122:3344:101::a]:63218","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":571,"response_code":"200"} {"msg":"disabling downstairs instance: downstairs-7a19008c-9c9c-4036-aa51-5a4ee7396154 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:22.0078583Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:22.082253863Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:22.092543835Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"region":"7a19008c-9c9c-4036-aa51-5a4ee7396154","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:22.146842885Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154","method":"GET","req_id":"cbd9c464-29d3-43e9-b13e-40dee2e3f119","remote_addr":"[fd00:1122:3344:101::a]:63218","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":94,"response_code":"200"} {"msg":"region 7a19008c-9c9c-4036-aa51-5a4ee7396154 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:22.170698759Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:25:22.304159028Z","hostname":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","pid":23789,"uri":"/crucible/0/regions/7a19008c-9c9c-4036-aa51-5a4ee7396154","method":"GET","req_id":"ab7bb0c5-f0e0-482d-9746-4afa83e02a0f","remote_addr":"[fd00:1122:3344:101::a]:63218","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":34,"response_code":"200"}