{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:09.93837805Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59105"}
{"msg":"region ffa35551-d867-4ea4-ac7f-f28cfa0363e2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:09.968711754Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:09.973184006Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions","method":"POST","req_id":"c1e3243c-ad78-486f-a762-452eccb6760a","remote_addr":"[fd00:1122:3344:101::a]:59105","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20770,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:09.978156652Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:09.98811833Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:09.98815027Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ffa35551-d867-4ea4-ac7f-f28cfa0363e2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.054551284Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"region":"ffa35551-d867-4ea4-ac7f-f28cfa0363e2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ffa35551-d867-4ea4-ac7f-f28cfa0363e2\" \"--data\" \"/data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.056586318Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"region":"ffa35551-d867-4ea4-ac7f-f28cfa0363e2","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.194523067Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"region":"ffa35551-d867-4ea4-ac7f-f28cfa0363e2","component":"worker"}
{"msg":"region ffa35551-d867-4ea4-ac7f-f28cfa0363e2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.194582187Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.194683068Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.195098493Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.201458107Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.220060994Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.223291872Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.234083838Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.236146942Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.236218113Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.236269263Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.240531633Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.24713642Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:01:10.285096953Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions","method":"POST","req_id":"a41f5789-be29-4334-a948-f99267a261df","remote_addr":"[fd00:1122:3344:101::a]:59105","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.810033144Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53726"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.813228502Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","method":"GET","req_id":"aae6f27b-c8cd-4fa7-bd1b-7cb39801f688","remote_addr":"[fd00:1122:3344:101::a]:53726","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":3172,"response_code":"200"}
{"msg":"path is \"/data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.816669492Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"dataset is oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.826400025Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"snapshot is oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2@951fdef0-d0dd-499c-b915-94460e51b7f8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.836031128Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2\", \"951fdef0-d0dd-499c-b915-94460e51b7f8\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.836070748Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.848494103Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2/snapshots/951fdef0-d0dd-499c-b915-94460e51b7f8","method":"GET","req_id":"32d1f7af-139a-48ae-8dce-05bb12a9b74f","remote_addr":"[fd00:1122:3344:101::a]:53726","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":31862,"response_code":"200"}
{"msg":"path is \"/data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.851581851Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"dataset is oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.86027039Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"snapshot is oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2@951fdef0-d0dd-499c-b915-94460e51b7f8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.869539769Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2\", \"951fdef0-d0dd-499c-b915-94460e51b7f8\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.869580239Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot ffa35551-d867-4ea4-ac7f-f28cfa0363e2-951fdef0-d0dd-499c-b915-94460e51b7f8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.879519665Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"applying SMF actions for region ffa35551-d867-4ea4-ac7f-f28cfa0363e2 running snapshot 951fdef0-d0dd-499c-b915-94460e51b7f8 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.880265334Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.880323934Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2/snapshots/951fdef0-d0dd-499c-b915-94460e51b7f8/run","method":"POST","req_id":"0654d7d5-1308-4115-886d-75118f83f7f3","remote_addr":"[fd00:1122:3344:101::a]:53726","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":28750,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.889538672Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.889623383Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.889702374Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.889709154Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.889750394Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.889797665Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.889894256Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.891857159Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.891911199Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.89196151Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.906780463Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-ffa35551-d867-4ea4-ac7f-f28cfa0363e2-951fdef0-d0dd-499c-b915-94460e51b7f8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.907343509Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-ffa35551-d867-4ea4-ac7f-f28cfa0363e2-951fdef0-d0dd-499c-b915-94460e51b7f8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.916767579Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-ffa35551-d867-4ea4-ac7f-f28cfa0363e2-951fdef0-d0dd-499c-b915-94460e51b7f8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.943957466Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2/.zfs/snapshot/951fdef0-d0dd-499c-b915-94460e51b7f8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.94427148Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.944331711Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.944374291Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.944414142Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.944463502Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.954199276Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.966058254Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"region ffa35551-d867-4ea4-ac7f-f28cfa0363e2 running snapshot 951fdef0-d0dd-499c-b915-94460e51b7f8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:27.966080524Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"path is \"/data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:28.185789356Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"dataset is oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:28.19554199Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"snapshot is oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2@951fdef0-d0dd-499c-b915-94460e51b7f8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:28.203382092Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2\", \"951fdef0-d0dd-499c-b915-94460e51b7f8\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:28.203418158Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:28.211032731Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2/snapshots/951fdef0-d0dd-499c-b915-94460e51b7f8/run","method":"POST","req_id":"2d7811ab-f453-4d12-bb56-b4b24950d819","remote_addr":"[fd00:1122:3344:101::a]:53726","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":25910,"response_code":"200"}
{"msg":"region 3344b611-208b-4ce3-a166-11e1a019851a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.738987257Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.739148199Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions","method":"POST","req_id":"2f71746e-9a6b-4e62-8555-a04b06c5b290","remote_addr":"[fd00:1122:3344:101::a]:53726","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":226,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.739177829Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/3344b611-208b-4ce3-a166-11e1a019851a","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.747338964Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/3344b611-208b-4ce3-a166-11e1a019851a","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.747363764Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3344b611-208b-4ce3-a166-11e1a019851a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3344b611-208b-4ce3-a166-11e1a019851a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.809098635Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"region":"3344b611-208b-4ce3-a166-11e1a019851a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3344b611-208b-4ce3-a166-11e1a019851a\" \"--data\" \"/data/regions/3344b611-208b-4ce3-a166-11e1a019851a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.809157965Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"region":"3344b611-208b-4ce3-a166-11e1a019851a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.950582905Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"region":"3344b611-208b-4ce3-a166-11e1a019851a","component":"worker"}
{"msg":"region 3344b611-208b-4ce3-a166-11e1a019851a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.950631995Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.950722836Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3344b611-208b-4ce3-a166-11e1a019851a","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.951171422Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3344b611-208b-4ce3-a166-11e1a019851a","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.95788543Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.967140538Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3344b611-208b-4ce3-a166-11e1a019851a","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.970429866Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3344b611-208b-4ce3-a166-11e1a019851a","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.97072225Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.97078549Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.970827971Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.970871471Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.975201162Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.980391944Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.980496177Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.980582559Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.980589318Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.980628432Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.980672194Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.985550203Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.985602673Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.985651504Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.985697474Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.989419368Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:31.991123228Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:32.091520908Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions","method":"POST","req_id":"4d070780-ad4b-46d6-a561-5063717ce5a5","remote_addr":"[fd00:1122:3344:101::a]:53726","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.284191945Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions/3344b611-208b-4ce3-a166-11e1a019851a","method":"GET","req_id":"1944a67b-a0f8-4864-9a50-9e3e2cb09784","remote_addr":"[fd00:1122:3344:101::a]:53726","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"path is \"/data/regions/3344b611-208b-4ce3-a166-11e1a019851a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.286826645Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"dataset is oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/3344b611-208b-4ce3-a166-11e1a019851a","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.298145648Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"region 3344b611-208b-4ce3-a166-11e1a019851a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.306471985Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.306660367Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.306729278Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions/3344b611-208b-4ce3-a166-11e1a019851a","method":"DELETE","req_id":"a70a3532-af6d-40bd-916e-c24b81c2c7ca","remote_addr":"[fd00:1122:3344:101::a]:53726","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19937,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-3344b611-208b-4ce3-a166-11e1a019851a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.30773782Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.309358438Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions/3344b611-208b-4ce3-a166-11e1a019851a","method":"GET","req_id":"db782953-cde4-49f8-87c4-e0dddbace0cb","remote_addr":"[fd00:1122:3344:101::a]:53726","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.313683509Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.31377158Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.313909361Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.313922992Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.313981612Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.314041523Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ffa35551-d867-4ea4-ac7f-f28cfa0363e2","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.314385457Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.314448108Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.314497128Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.314544329Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.318334003Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.319899391Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_3634a2c6-2441-44d1-961c-492b8f9a328b/crucible/regions/3344b611-208b-4ce3-a166-11e1a019851a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.328195558Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"region":"3344b611-208b-4ce3-a166-11e1a019851a","component":"worker"}
{"msg":"region 3344b611-208b-4ce3-a166-11e1a019851a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.403635128Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-09T21:06:57.450755628Z","hostname":"oxz_crucible_86f3407d-fb85-4c2e-a338-33b389f713e9","pid":21772,"uri":"/crucible/0/regions/3344b611-208b-4ce3-a166-11e1a019851a","method":"GET","req_id":"6f4de59b-cc65-4c0a-93cf-1600f69c329e","remote_addr":"[fd00:1122:3344:101::a]:53726","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":76,"response_code":"200"}