[ Mar  1 23:17:33 Enabled. ]
[ Mar  1 23:17:33 Rereading configuration. ]
[ Mar  1 23:17:38 Rereading configuration. ]
[ Mar  1 23:17:43 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:43.971700768Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:43.975641775Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:43.975654465Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:17:44.282389389Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.414510125Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54933"}
{"msg":"region 1136a670-e0c2-4677-a2a0-e27bf38aad53 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.48046344Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.483107195Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"uri":"/crucible/0/regions","method":"POST","req_id":"0744e906-babb-4b96-b237-04bfc8d4994e","remote_addr":"[fd00:1122:3344:101::a]:54933","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":5757,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.483757211Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.495232412Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.495267593Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1136a670-e0c2-4677-a2a0-e27bf38aad53\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.54657322Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"region":"1136a670-e0c2-4677-a2a0-e27bf38aad53","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.620623314Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"uri":"/crucible/0/regions","method":"POST","req_id":"552f1238-d2ac-40a0-8e50-76ff7d3c7a4e","remote_addr":"[fd00:1122:3344:101::a]:54933","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":562,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.684487518Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"region":"1136a670-e0c2-4677-a2a0-e27bf38aad53","component":"worker"}
{"msg":"region 1136a670-e0c2-4677-a2a0-e27bf38aad53 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.684531929Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.684673953Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1136a670-e0c2-4677-a2a0-e27bf38aad53","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.685152945Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1136a670-e0c2-4677-a2a0-e27bf38aad53","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.690092875Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.70088007Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1136a670-e0c2-4677-a2a0-e27bf38aad53","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.704284643Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.705104363Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.705239557Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.705281738Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.705321468Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.708911156Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.713259663Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:19:07.985032651Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"uri":"/crucible/0/regions","method":"POST","req_id":"47e8c853-51af-49bb-8688-d66d91f3eb1f","remote_addr":"[fd00:1122:3344:101::a]:54933","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.097891551Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54576"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.101597392Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"uri":"/crucible/0/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53","method":"GET","req_id":"80510b78-3bed-4788-9080-0bd01c181473","remote_addr":"[fd00:1122:3344:101::a]:54576","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":3643,"response_code":"200"}
{"msg":"path is \"/data/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.104150685Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"datafile"}
{"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.113265828Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"datafile"}
{"msg":"snapshot is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53@8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.122110795Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53\", \"8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.122154226Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.138536037Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"uri":"/crucible/0/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53/snapshots/8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","method":"GET","req_id":"c6508b12-2057-46c1-a05f-ba9b91f9a82f","remote_addr":"[fd00:1122:3344:101::a]:54576","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":34405,"response_code":"200"}
{"msg":"path is \"/data/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.141007808Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"datafile"}
{"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.150001208Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"datafile"}
{"msg":"snapshot is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53@8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.157787929Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53\", \"8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.15781431Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 1136a670-e0c2-4677-a2a0-e27bf38aad53-8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.174009606Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"datafile"}
{"msg":"applying SMF actions for region 1136a670-e0c2-4677-a2a0-e27bf38aad53 running snapshot 8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.174810256Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.174839267Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"uri":"/crucible/0/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53/snapshots/8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0/run","method":"POST","req_id":"50b4399c-8a95-417b-918e-862d06a401af","remote_addr":"[fd00:1122:3344:101::a]:54576","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":33866,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-1136a670-e0c2-4677-a2a0-e27bf38aad53-8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.181469449Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-1136a670-e0c2-4677-a2a0-e27bf38aad53-8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.193654507Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-1136a670-e0c2-4677-a2a0-e27bf38aad53-8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.224858542Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1136a670-e0c2-4677-a2a0-e27bf38aad53/.zfs/snapshot/8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.22520468Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.225303453Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.225392545Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.225460836Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.225508658Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.240737931Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.256136528Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"worker"}
{"msg":"region 1136a670-e0c2-4677-a2a0-e27bf38aad53 running snapshot 8e91b2a9-31e6-4d8a-8ad3-322b7a218ff0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-01T23:24:31.256163989Z","hostname":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","pid":23580,"component":"datafile"}