[ Oct  3 20:31:48 Enabled. ]
[ Oct  3 20:31:48 Rereading configuration. ]
[ Oct  3 20:31:50 Rereading configuration. ]
[ Oct  3 20:31:52 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:31:53.024989923Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:31:53.026564812Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:31:53.026575642Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:31:53.044661612Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:31:53.288135963Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.269612749Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35797"}
{"msg":"region f2591d0a-310c-4551-b6dd-7ca41cdb0016 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.304095287Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.308437995Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions","method":"POST","req_id":"45f924e6-a52b-471c-b74d-c8ffd29f0878","remote_addr":"[fd00:1122:3344:101::a]:35797","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":12303,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.31223914Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.321357687Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.321391898Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f2591d0a-310c-4551-b6dd-7ca41cdb0016\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.378902129Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"region":"f2591d0a-310c-4551-b6dd-7ca41cdb0016","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f2591d0a-310c-4551-b6dd-7ca41cdb0016\" \"--data\" \"/data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.382691614Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"region":"f2591d0a-310c-4551-b6dd-7ca41cdb0016","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.446529303Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions","method":"POST","req_id":"22f2708a-349d-4fab-a758-09881db7fd15","remote_addr":"[fd00:1122:3344:101::a]:35797","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.677766608Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"region":"f2591d0a-310c-4551-b6dd-7ca41cdb0016","component":"worker"}
{"msg":"region f2591d0a-310c-4551-b6dd-7ca41cdb0016 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.677809409Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.677909342Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.680409694Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.684348412Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.701796267Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.705308974Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.713954129Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.714021071Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.714065182Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.714104343Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.717544029Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.721340003Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:33:08.792206997Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions","method":"POST","req_id":"3711b158-51b3-46ba-82a7-e9a408de796a","remote_addr":"[fd00:1122:3344:101::a]:35797","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.875038415Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37631"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.878578103Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","method":"GET","req_id":"3250a41c-8aac-4115-8ea8-5ae0eccd8672","remote_addr":"[fd00:1122:3344:101::a]:37631","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":3430,"response_code":"200"}
{"msg":"path is \"/data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.882379598Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"dataset is oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.892281574Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"snapshot is oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016@bec3d930-a591-451d-82e5-be08acca0f21","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.901178176Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016\", \"bec3d930-a591-451d-82e5-be08acca0f21\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.901212597Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.914531108Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016/snapshots/bec3d930-a591-451d-82e5-be08acca0f21","method":"GET","req_id":"131f9944-0d43-44a8-a337-5a93b19f01ab","remote_addr":"[fd00:1122:3344:101::a]:37631","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":32788,"response_code":"200"}
{"msg":"path is \"/data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.917696477Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"dataset is oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.925295596Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"snapshot is oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016@bec3d930-a591-451d-82e5-be08acca0f21","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.933216243Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016\", \"bec3d930-a591-451d-82e5-be08acca0f21\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.933244344Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot f2591d0a-310c-4551-b6dd-7ca41cdb0016-bec3d930-a591-451d-82e5-be08acca0f21 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.941818117Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"applying SMF actions for region f2591d0a-310c-4551-b6dd-7ca41cdb0016 running snapshot bec3d930-a591-451d-82e5-be08acca0f21 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.942457033Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.942478924Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016/snapshots/bec3d930-a591-451d-82e5-be08acca0f21/run","method":"POST","req_id":"8e7b4dd6-18de-44e8-979b-2a1e632c279d","remote_addr":"[fd00:1122:3344:101::a]:37631","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24778,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.94956697Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.949651362Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.949746934Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.949765825Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.949797606Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.949842047Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.949936969Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.949989811Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.950037002Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.950093793Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.959879846Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-f2591d0a-310c-4551-b6dd-7ca41cdb0016-bec3d930-a591-451d-82e5-be08acca0f21","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.960320617Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-f2591d0a-310c-4551-b6dd-7ca41cdb0016-bec3d930-a591-451d-82e5-be08acca0f21","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.96363275Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-f2591d0a-310c-4551-b6dd-7ca41cdb0016-bec3d930-a591-451d-82e5-be08acca0f21","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.975521835Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016/.zfs/snapshot/bec3d930-a591-451d-82e5-be08acca0f21","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.975776442Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.975841253Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.975882364Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.975923585Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.975977837Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.980761006Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.986014596Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"region f2591d0a-310c-4551-b6dd-7ca41cdb0016 running snapshot bec3d930-a591-451d-82e5-be08acca0f21 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:12.986032117Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"path is \"/data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:13.218099739Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"dataset is oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:13.227271622Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"snapshot is oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016@bec3d930-a591-451d-82e5-be08acca0f21","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:13.235184009Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016\", \"bec3d930-a591-451d-82e5-be08acca0f21\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:13.23523573Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:13.242661095Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016/snapshots/bec3d930-a591-451d-82e5-be08acca0f21/run","method":"POST","req_id":"67b668e9-5263-45a4-95f6-afe17e07dde5","remote_addr":"[fd00:1122:3344:101::a]:37631","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24559,"response_code":"200"}
{"msg":"region 025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.144871334Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.145041929Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions","method":"POST","req_id":"6854dc08-d3f3-4c6b-b8c5-4dcb32254fc1","remote_addr":"[fd00:1122:3344:101::a]:37631","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":249,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.145071569Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.155248612Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.155284553Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.236030503Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"region":"025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c\" \"--data\" \"/data/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.236094225Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"region":"025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.372025368Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"region":"025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","component":"worker"}
{"msg":"region 025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.372068489Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.372162812Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.372602613Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.379604477Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.393929904Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.400032916Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.400346553Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.400401375Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.400443216Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.400489237Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.408424814Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.418584407Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.418717571Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.418837944Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.418847424Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.418900875Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.418957657Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.425871469Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.425954381Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.426019552Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.426091104Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.433008026Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.435115419Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:17.441950089Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions","method":"POST","req_id":"f7e0e0af-6b94-4674-91ab-ca2e3a1444ed","remote_addr":"[fd00:1122:3344:101::a]:37631","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":615,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.112806428Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","method":"GET","req_id":"64cb5fbb-d53c-405f-8457-9e06aa4e9fc8","remote_addr":"[fd00:1122:3344:101::a]:37631","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"path is \"/data/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.116601243Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"dataset is oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.131599936Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"region 025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.141756629Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.141983345Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.143120113Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.144450916Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","method":"DELETE","req_id":"0ebc6c41-2c79-4417-bf74-02aa3a6c67d4","remote_addr":"[fd00:1122:3344:101::a]:37631","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27902,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.148460746Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","method":"GET","req_id":"38012a2c-c622-477e-a29e-10a5d3851d81","remote_addr":"[fd00:1122:3344:101::a]:37631","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":525,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.148911097Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.14900254Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.149100592Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.149120463Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.149170944Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.149238775Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f2591d0a-310c-4551-b6dd-7ca41cdb0016","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.149573694Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.149657236Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.149722338Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.149774479Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.154321322Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.15585318Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7f092e33-1af0-4e4e-87b6-07db26b26d4d/crucible/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.167055829Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"region":"025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","component":"worker"}
{"msg":"region 025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.230048057Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-03T20:36:42.287247731Z","hostname":"oxz_crucible_02b46aad-d593-4b9f-86d0-f084daa4a4f0","pid":21735,"uri":"/crucible/0/regions/025e3246-a1bf-4cfd-91d6-a68f4ef7cb2c","method":"GET","req_id":"7ceecb7c-5c0c-4e90-a833-fd92dbe584d7","remote_addr":"[fd00:1122:3344:101::a]:37631","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":772,"response_code":"200"}