{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.601631162Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56553"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.601725253Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"uri":"/crucible/0/regions/c13a88b5-9eb1-4029-9582-3986824ad027","method":"GET","req_id":"7653937b-9718-4073-a0d4-0a266fb4ca62","remote_addr":"[fd00:1122:3344:101::a]:56553","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"path is \"/data/regions/c13a88b5-9eb1-4029-9582-3986824ad027\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.60701692Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/c13a88b5-9eb1-4029-9582-3986824ad027","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.618801665Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"snapshot is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/c13a88b5-9eb1-4029-9582-3986824ad027@4d67639d-fe66-4540-b335-76f647063e02","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.628044537Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/c13a88b5-9eb1-4029-9582-3986824ad027\", \"4d67639d-fe66-4540-b335-76f647063e02\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.628070858Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.639970244Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"uri":"/crucible/0/regions/c13a88b5-9eb1-4029-9582-3986824ad027/snapshots/4d67639d-fe66-4540-b335-76f647063e02","method":"GET","req_id":"0df7e141-81e0-4642-a2d5-ed230ae35c8f","remote_addr":"[fd00:1122:3344:101::a]:56553","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":38088,"response_code":"200"}
{"msg":"path is \"/data/regions/c13a88b5-9eb1-4029-9582-3986824ad027\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.640149567Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/c13a88b5-9eb1-4029-9582-3986824ad027","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.648658717Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"snapshot is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/c13a88b5-9eb1-4029-9582-3986824ad027@4d67639d-fe66-4540-b335-76f647063e02","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.657038495Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/c13a88b5-9eb1-4029-9582-3986824ad027\", \"4d67639d-fe66-4540-b335-76f647063e02\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.657063525Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot c13a88b5-9eb1-4029-9582-3986824ad027-4d67639d-fe66-4540-b335-76f647063e02 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.667328185Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"applying SMF actions for region c13a88b5-9eb1-4029-9582-3986824ad027 running snapshot 4d67639d-fe66-4540-b335-76f647063e02 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.667891464Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.667920384Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"uri":"/crucible/0/regions/c13a88b5-9eb1-4029-9582-3986824ad027/snapshots/4d67639d-fe66-4540-b335-76f647063e02/run","method":"POST","req_id":"8f58d53a-ae0c-41c1-a6a3-abd391f91962","remote_addr":"[fd00:1122:3344:101::a]:56553","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":27806,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-c13a88b5-9eb1-4029-9582-3986824ad027-4d67639d-fe66-4540-b335-76f647063e02","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.673977904Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-c13a88b5-9eb1-4029-9582-3986824ad027-4d67639d-fe66-4540-b335-76f647063e02","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.679595697Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-c13a88b5-9eb1-4029-9582-3986824ad027-4d67639d-fe66-4540-b335-76f647063e02","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.691556454Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c13a88b5-9eb1-4029-9582-3986824ad027/.zfs/snapshot/4d67639d-fe66-4540-b335-76f647063e02","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.691848969Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.691903779Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.69194752Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.691986541Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.692037132Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.696281802Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.701319851Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"region c13a88b5-9eb1-4029-9582-3986824ad027 running snapshot 4d67639d-fe66-4540-b335-76f647063e02 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:07.701328608Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.106596975Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34630"}
{"msg":"region e714813c-0656-466a-90ca-1398998e34af state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.166462541Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.166612494Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"uri":"/crucible/0/regions","method":"POST","req_id":"7ea38e82-4bda-4bdc-9a56-eb4ae445729a","remote_addr":"[fd00:1122:3344:101::a]:34630","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":1009,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.166631584Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/e714813c-0656-466a-90ca-1398998e34af","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.175825316Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/e714813c-0656-466a-90ca-1398998e34af","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.175862376Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e714813c-0656-466a-90ca-1398998e34af\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/e714813c-0656-466a-90ca-1398998e34af\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.231121767Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"region":"e714813c-0656-466a-90ca-1398998e34af","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.334127364Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"region":"e714813c-0656-466a-90ca-1398998e34af","component":"worker"}
{"msg":"region e714813c-0656-466a-90ca-1398998e34af state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.334185536Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.334284287Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e714813c-0656-466a-90ca-1398998e34af","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.335628209Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e714813c-0656-466a-90ca-1398998e34af","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.34415354Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.3520389Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e714813c-0656-466a-90ca-1398998e34af","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.355430326Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e714813c-0656-466a-90ca-1398998e34af","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.35572966Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.355785692Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.355841692Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.355883893Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.359435701Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.365543602Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:10.396129956Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"uri":"/crucible/0/regions","method":"POST","req_id":"61b7ad4e-ff94-4eb5-8de5-e804fc6496ec","remote_addr":"[fd00:1122:3344:101::a]:34630","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.444354438Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62422"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.444425849Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"uri":"/crucible/0/regions/e714813c-0656-466a-90ca-1398998e34af","method":"GET","req_id":"aba5d230-e526-4c11-9e4a-877e4a021885","remote_addr":"[fd00:1122:3344:101::a]:62422","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"path is \"/data/regions/e714813c-0656-466a-90ca-1398998e34af\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.444664013Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/e714813c-0656-466a-90ca-1398998e34af","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.457151209Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"region e714813c-0656-466a-90ca-1398998e34af state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.466290809Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.466504993Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.466548303Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"uri":"/crucible/0/regions/e714813c-0656-466a-90ca-1398998e34af","method":"DELETE","req_id":"2ae6ed9b-6697-44a8-8b86-5a53e692cab7","remote_addr":"[fd00:1122:3344:101::a]:62422","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21927,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.466741517Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"uri":"/crucible/0/regions/e714813c-0656-466a-90ca-1398998e34af","method":"GET","req_id":"2ff237d2-6602-4cb4-b0bf-5591ead53f02","remote_addr":"[fd00:1122:3344:101::a]:62422","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-e714813c-0656-466a-90ca-1398998e34af (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.467740973Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.47485228Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/e714813c-0656-466a-90ca-1398998e34af\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.483831588Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"region":"e714813c-0656-466a-90ca-1398998e34af","component":"worker"}
{"msg":"region e714813c-0656-466a-90ca-1398998e34af state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.534012415Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-16T03:03:42.584099501Z","hostname":"oxz_crucible_ef3802f7-87fc-4b1d-b86d-f47aacb298fa","pid":22336,"uri":"/crucible/0/regions/e714813c-0656-466a-90ca-1398998e34af","method":"GET","req_id":"0a9719d8-5d9e-4bb6-a65e-2911bdeb40de","remote_addr":"[fd00:1122:3344:101::a]:62422","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":104,"response_code":"200"}