[ Feb 22 18:31:16 Enabled. ]
[ Feb 22 18:31:16 Rereading configuration. ]
[ Feb 22 18:31:18 Rereading configuration. ]
[ Feb 22 18:31:28 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:28.731654542Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:28.735285403Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:28.735297853Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:31:28.981913683Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:29.955383541Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56574"}
{"msg":"region 8626f0ee-549f-4fad-9c87-482b61b90830 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.02401692Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.026231133Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions","method":"POST","req_id":"c02e885f-c7ac-423f-a754-d255f92ee4bc","remote_addr":"[fd00:1122:3344:101::a]:56574","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":5155,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.026924687Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/8626f0ee-549f-4fad-9c87-482b61b90830","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.035738799Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/8626f0ee-549f-4fad-9c87-482b61b90830","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.035773389Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8626f0ee-549f-4fad-9c87-482b61b90830\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/8626f0ee-549f-4fad-9c87-482b61b90830\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.081737416Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"region":"8626f0ee-549f-4fad-9c87-482b61b90830","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.206915707Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"region":"8626f0ee-549f-4fad-9c87-482b61b90830","component":"worker"}
{"msg":"region 8626f0ee-549f-4fad-9c87-482b61b90830 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.206967408Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.207044429Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8626f0ee-549f-4fad-9c87-482b61b90830","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.207432017Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8626f0ee-549f-4fad-9c87-482b61b90830","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.213625638Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.223362067Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8626f0ee-549f-4fad-9c87-482b61b90830","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.226459008Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8626f0ee-549f-4fad-9c87-482b61b90830","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.227192162Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.227280704Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.227329725Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.227370986Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.231609378Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.236597216Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:32:30.240253217Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions","method":"POST","req_id":"8c7b769e-587d-4df8-9174-740441ea34a4","remote_addr":"[fd00:1122:3344:101::a]:56574","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.664261152Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:38742"}
{"msg":"region 3d7b769e-479a-4386-96ce-0f08d2587518 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.664903725Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.665559787Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions","method":"POST","req_id":"b7499113-a1c3-477a-985e-baa11502bfca","remote_addr":"[fd00:1122:3344:101::a]:38742","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":1280,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.665580988Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/3d7b769e-479a-4386-96ce-0f08d2587518","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.674188646Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/3d7b769e-479a-4386-96ce-0f08d2587518","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.674218606Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3d7b769e-479a-4386-96ce-0f08d2587518\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/3d7b769e-479a-4386-96ce-0f08d2587518\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.727840322Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"region":"3d7b769e-479a-4386-96ce-0f08d2587518","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.796868528Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"region":"3d7b769e-479a-4386-96ce-0f08d2587518","component":"worker"}
{"msg":"region 3d7b769e-479a-4386-96ce-0f08d2587518 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.796929749Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.797014601Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3d7b769e-479a-4386-96ce-0f08d2587518","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.79748077Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3d7b769e-479a-4386-96ce-0f08d2587518","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.803070839Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.803289163Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions","method":"POST","req_id":"3d427879-492b-419d-ae84-0476cb6f23a3","remote_addr":"[fd00:1122:3344:101::a]:38742","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.815749556Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3d7b769e-479a-4386-96ce-0f08d2587518","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.818563411Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3d7b769e-479a-4386-96ce-0f08d2587518","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.818884758Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.818939068Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.818979219Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.81901794Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.82211529Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:50.827290402Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.018352139Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61977"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.02095107Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions/8626f0ee-549f-4fad-9c87-482b61b90830","method":"GET","req_id":"9b0ff923-82f9-41d4-9ff0-85b164a66c95","remote_addr":"[fd00:1122:3344:101::a]:61977","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":2570,"response_code":"200"}
{"msg":"path is \"/data/regions/8626f0ee-549f-4fad-9c87-482b61b90830\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.021100983Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"dataset is oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/8626f0ee-549f-4fad-9c87-482b61b90830","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.030821282Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"snapshot is oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/8626f0ee-549f-4fad-9c87-482b61b90830@5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.039791428Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/8626f0ee-549f-4fad-9c87-482b61b90830\", \"5d9df203-ce89-4f7f-9ce1-08b8316a4558\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.039823258Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.049831353Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions/8626f0ee-549f-4fad-9c87-482b61b90830/snapshots/5d9df203-ce89-4f7f-9ce1-08b8316a4558","method":"GET","req_id":"eb5572d1-3d71-46ff-9fae-b616f63abe24","remote_addr":"[fd00:1122:3344:101::a]:61977","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":28736,"response_code":"200"}
{"msg":"path is \"/data/regions/8626f0ee-549f-4fad-9c87-482b61b90830\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.049995607Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"dataset is oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/8626f0ee-549f-4fad-9c87-482b61b90830","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.058462612Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"snapshot is oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/8626f0ee-549f-4fad-9c87-482b61b90830@5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.06709888Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/8626f0ee-549f-4fad-9c87-482b61b90830\", \"5d9df203-ce89-4f7f-9ce1-08b8316a4558\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.067137241Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 8626f0ee-549f-4fad-9c87-482b61b90830-5d9df203-ce89-4f7f-9ce1-08b8316a4558 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.075974223Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"applying SMF actions for region 8626f0ee-549f-4fad-9c87-482b61b90830 running snapshot 5d9df203-ce89-4f7f-9ce1-08b8316a4558 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.076138476Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.076193477Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions/8626f0ee-549f-4fad-9c87-482b61b90830/snapshots/5d9df203-ce89-4f7f-9ce1-08b8316a4558/run","method":"POST","req_id":"2b59c7fa-c2da-43fa-a46d-fa83a1893fbb","remote_addr":"[fd00:1122:3344:101::a]:61977","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":26225,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-8626f0ee-549f-4fad-9c87-482b61b90830-5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.078506943Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-8626f0ee-549f-4fad-9c87-482b61b90830-5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.086640161Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-8626f0ee-549f-4fad-9c87-482b61b90830-5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.098252368Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8626f0ee-549f-4fad-9c87-482b61b90830/.zfs/snapshot/5d9df203-ce89-4f7f-9ce1-08b8316a4558","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.098512123Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.098571674Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.098613385Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.098663636Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.098716817Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.102632013Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.106945317Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"region 8626f0ee-549f-4fad-9c87-482b61b90830 running snapshot 5d9df203-ce89-4f7f-9ce1-08b8316a4558 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:52.106961218Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.774511557Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42212"}
{"msg":"region c94843fe-a7b0-4b90-883d-d36016f76c26 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.774626229Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.774895515Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions","method":"POST","req_id":"f959d025-7cb3-44e6-ac65-6b412417a546","remote_addr":"[fd00:1122:3344:101::a]:42212","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":348,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.774924345Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/c94843fe-a7b0-4b90-883d-d36016f76c26","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.783566564Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/c94843fe-a7b0-4b90-883d-d36016f76c26","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.783592934Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c94843fe-a7b0-4b90-883d-d36016f76c26\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/c94843fe-a7b0-4b90-883d-d36016f76c26\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.849087552Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"region":"c94843fe-a7b0-4b90-883d-d36016f76c26","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.913704242Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"region":"c94843fe-a7b0-4b90-883d-d36016f76c26","component":"worker"}
{"msg":"region c94843fe-a7b0-4b90-883d-d36016f76c26 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.913760073Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.913873985Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c94843fe-a7b0-4b90-883d-d36016f76c26","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.915898184Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c94843fe-a7b0-4b90-883d-d36016f76c26","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.918961194Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.926904339Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c94843fe-a7b0-4b90-883d-d36016f76c26","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.930336026Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c94843fe-a7b0-4b90-883d-d36016f76c26","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.930603271Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.930666712Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.930712313Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.930756334Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.934506407Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:54.939890772Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:37:55.002624966Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions","method":"POST","req_id":"bf840491-f12d-4160-b2e6-e46b41b4c653","remote_addr":"[fd00:1122:3344:101::a]:42212","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.437802347Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37510"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.437877769Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions/c94843fe-a7b0-4b90-883d-d36016f76c26","method":"GET","req_id":"3f7a9082-4e47-435f-ac84-0a05adccc3de","remote_addr":"[fd00:1122:3344:101::a]:37510","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"path is \"/data/regions/c94843fe-a7b0-4b90-883d-d36016f76c26\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.438064022Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"dataset is oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/c94843fe-a7b0-4b90-883d-d36016f76c26","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.449369883Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"region c94843fe-a7b0-4b90-883d-d36016f76c26 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.460871607Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.461064201Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.461127612Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions/c94843fe-a7b0-4b90-883d-d36016f76c26","method":"DELETE","req_id":"09caddc3-fc24-4706-af5b-4cdbd7282193","remote_addr":"[fd00:1122:3344:101::a]:37510","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":23102,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.461856316Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions/c94843fe-a7b0-4b90-883d-d36016f76c26","method":"GET","req_id":"56b7a7bb-12d6-4bff-af7f-b657ca7ce1bc","remote_addr":"[fd00:1122:3344:101::a]:37510","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":582,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-c94843fe-a7b0-4b90-883d-d36016f76c26 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.462297965Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.469519226Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crucible/regions/c94843fe-a7b0-4b90-883d-d36016f76c26\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.479275426Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"region":"c94843fe-a7b0-4b90-883d-d36016f76c26","component":"worker"}
{"msg":"region c94843fe-a7b0-4b90-883d-d36016f76c26 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.526008177Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T18:38:25.563150382Z","hostname":"oxz_crucible_bb345b44-340b-4df2-aae1-2316d65631f9","pid":26420,"uri":"/crucible/0/regions/c94843fe-a7b0-4b90-883d-d36016f76c26","method":"GET","req_id":"0ab57dc5-0a1a-47ec-814b-6ca1115f513c","remote_addr":"[fd00:1122:3344:101::a]:37510","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":60,"response_code":"200"}