[ Jul 25 05:48:05 Enabled. ]
[ Jul 25 05:48:06 Rereading configuration. ]
[ Jul 25 05:48:10 Rereading configuration. ]
[ Jul 25 05:48:13 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:13.794120593Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:13.801133959Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:13.80115103Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:13.822798334Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:13.97971362Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.257086539Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48751"}
{"msg":"region 473e6bbf-37d3-4fca-9547-a231d4e332b4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.282581215Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.290185776Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions","method":"POST","req_id":"9601a5f2-a6bc-493b-ac4a-d83db88ee03b","remote_addr":"[fd00:1122:3344:101::a]:48751","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18596,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.292875359Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.304005974Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.304036625Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"473e6bbf-37d3-4fca-9547-a231d4e332b4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.359289106Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"region":"473e6bbf-37d3-4fca-9547-a231d4e332b4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"473e6bbf-37d3-4fca-9547-a231d4e332b4\" \"--data\" \"/data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.361640692Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"region":"473e6bbf-37d3-4fca-9547-a231d4e332b4","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.603364521Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"region":"473e6bbf-37d3-4fca-9547-a231d4e332b4","component":"worker"}
{"msg":"region 473e6bbf-37d3-4fca-9547-a231d4e332b4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.603426893Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.603516125Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.604005587Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.611359841Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.615313715Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions","method":"POST","req_id":"addd7711-914d-4ab8-bb92-2e733fe64bf6","remote_addr":"[fd00:1122:3344:101::a]:48751","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.627431193Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.630622359Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.631387407Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.636464367Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.636539639Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.63658685Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.641783274Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:49:26.649295442Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.546496639Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34554"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.55449716Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","method":"GET","req_id":"cd48796d-da01-4b2d-8bd2-45f7ed024495","remote_addr":"[fd00:1122:3344:101::a]:34554","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":7963,"response_code":"200"}
{"msg":"path is \"/data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.556830895Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"dataset is oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.566838502Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"snapshot is oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4@02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.575945199Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4\", \"02c17e09-5830-4ee8-a416-a66468634fe2\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.575971339Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.59158503Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4/snapshots/02c17e09-5830-4ee8-a416-a66468634fe2","method":"GET","req_id":"d508e34e-3ae6-46ef-85ca-18adfceee3fa","remote_addr":"[fd00:1122:3344:101::a]:34554","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":36915,"response_code":"200"}
{"msg":"path is \"/data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.591754604Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"dataset is oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.59999803Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"snapshot is oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4@02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.607650462Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4\", \"02c17e09-5830-4ee8-a416-a66468634fe2\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.607686912Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 473e6bbf-37d3-4fca-9547-a231d4e332b4-02c17e09-5830-4ee8-a416-a66468634fe2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.616059181Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"applying SMF actions for region 473e6bbf-37d3-4fca-9547-a231d4e332b4 running snapshot 02c17e09-5830-4ee8-a416-a66468634fe2 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.621544301Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.621610123Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4/snapshots/02c17e09-5830-4ee8-a416-a66468634fe2/run","method":"POST","req_id":"97de1e72-2653-4270-b148-00a2b62be883","remote_addr":"[fd00:1122:3344:101::a]:34554","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":29883,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.632522322Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.632609394Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.632690306Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.632698197Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.632739527Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.632787228Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.632890641Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.632951932Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.632998213Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.633057295Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.660685091Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-473e6bbf-37d3-4fca-9547-a231d4e332b4-02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.661176413Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-473e6bbf-37d3-4fca-9547-a231d4e332b4-02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.688205975Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-473e6bbf-37d3-4fca-9547-a231d4e332b4-02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.759153479Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4/.zfs/snapshot/02c17e09-5830-4ee8-a416-a66468634fe2","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.759469717Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.759532018Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.759583359Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.759627211Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.759678052Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.786004347Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.801738191Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"region 473e6bbf-37d3-4fca-9547-a231d4e332b4 running snapshot 02c17e09-5830-4ee8-a416-a66468634fe2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:54.801775841Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.617293234Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57253"}
{"msg":"region 497052a2-77a0-47df-a927-ed5dda7e9049 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.617368866Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.621386461Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.621397481Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions","method":"POST","req_id":"f99ed8cd-53ac-45e8-8088-2bb84d03355e","remote_addr":"[fd00:1122:3344:101::a]:57253","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":4073,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/497052a2-77a0-47df-a927-ed5dda7e9049","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.630764804Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/497052a2-77a0-47df-a927-ed5dda7e9049","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.630790945Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"497052a2-77a0-47df-a927-ed5dda7e9049\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/497052a2-77a0-47df-a927-ed5dda7e9049\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.705764475Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"region":"497052a2-77a0-47df-a927-ed5dda7e9049","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"497052a2-77a0-47df-a927-ed5dda7e9049\" \"--data\" \"/data/regions/497052a2-77a0-47df-a927-ed5dda7e9049\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.705818906Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"region":"497052a2-77a0-47df-a927-ed5dda7e9049","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.838980568Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions","method":"POST","req_id":"5267a516-5972-4fcd-98bc-0b63874e012e","remote_addr":"[fd00:1122:3344:101::a]:57253","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.914190294Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"region":"497052a2-77a0-47df-a927-ed5dda7e9049","component":"worker"}
{"msg":"region 497052a2-77a0-47df-a927-ed5dda7e9049 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.914221525Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.914359658Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.915013054Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.915113326Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.915211199Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.915219329Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.91526933Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.915329332Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.91568301Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.915741521Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.915790322Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.915837984Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.928902584Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-497052a2-77a0-47df-a927-ed5dda7e9049","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.929446757Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-497052a2-77a0-47df-a927-ed5dda7e9049","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.941041052Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.970225765Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-497052a2-77a0-47df-a927-ed5dda7e9049","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.976938024Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/497052a2-77a0-47df-a927-ed5dda7e9049","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.977238631Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.977317064Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.977364534Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.977404986Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:58.994079111Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:59.01338607Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:59.244618971Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions","method":"POST","req_id":"700819e2-8055-4443-bf56-386a2f55c871","remote_addr":"[fd00:1122:3344:101::a]:57253","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":632,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.803974073Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions/497052a2-77a0-47df-a927-ed5dda7e9049","method":"GET","req_id":"6d23dbc8-80e7-4868-b7a3-89ac06e063cf","remote_addr":"[fd00:1122:3344:101::a]:57253","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"path is \"/data/regions/497052a2-77a0-47df-a927-ed5dda7e9049\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.806588145Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"dataset is oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/497052a2-77a0-47df-a927-ed5dda7e9049","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.832356597Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"region 497052a2-77a0-47df-a927-ed5dda7e9049 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.841072624Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.841273219Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.841348401Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions/497052a2-77a0-47df-a927-ed5dda7e9049","method":"DELETE","req_id":"f1652f12-eb84-4971-997d-90835dfdb079","remote_addr":"[fd00:1122:3344:101::a]:57253","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":34785,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-497052a2-77a0-47df-a927-ed5dda7e9049 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.842847486Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.844539846Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions/497052a2-77a0-47df-a927-ed5dda7e9049","method":"GET","req_id":"8fde695f-b240-429f-9349-ff8696403c8b","remote_addr":"[fd00:1122:3344:101::a]:57253","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":543,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.858706023Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.858828396Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.858930238Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.858954719Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.85898436Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.859052241Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/473e6bbf-37d3-4fca-9547-a231d4e332b4","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.859687306Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.859783499Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.85985061Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.859922152Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.892062455Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.893636452Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crucible/regions/497052a2-77a0-47df-a927-ed5dda7e9049\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.902929293Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"region":"497052a2-77a0-47df-a927-ed5dda7e9049","component":"worker"}
{"msg":"region 497052a2-77a0-47df-a927-ed5dda7e9049 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.973450498Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:55:26.995651665Z","hostname":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","pid":28580,"uri":"/crucible/0/regions/497052a2-77a0-47df-a927-ed5dda7e9049","method":"GET","req_id":"c08a3dec-4ee5-4e9f-88b5-4110db5b044f","remote_addr":"[fd00:1122:3344:101::a]:57253","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":87,"response_code":"200"}