{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.842025318Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49264"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.844986736Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"uri":"/crucible/0/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881","method":"GET","req_id":"c1a8d1f4-d163-43ba-8d66-694f7151438f","remote_addr":"[fd00:1122:3344:101::a]:49264","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":2921,"response_code":"200"} {"msg":"path is \"/data/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.848348773Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"datafile"} {"msg":"dataset is oxp_4a0f3222-7021-44c5-8399-6c21f4349e5e/crucible/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.857766169Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"datafile"} {"msg":"snapshot is oxp_4a0f3222-7021-44c5-8399-6c21f4349e5e/crucible/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881@17b5f014-6b60-49c6-a475-273fd6651c09","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.865541137Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_4a0f3222-7021-44c5-8399-6c21f4349e5e/crucible/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881\", \"17b5f014-6b60-49c6-a475-273fd6651c09\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.865573928Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.877076192Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"uri":"/crucible/0/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881/snapshots/17b5f014-6b60-49c6-a475-273fd6651c09","method":"GET","req_id":"76197296-f34b-471f-9bcf-0dd21de55e05","remote_addr":"[fd00:1122:3344:101::a]:49264","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":28736,"response_code":"200"} {"msg":"path is \"/data/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.880695665Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"datafile"} {"msg":"dataset is oxp_4a0f3222-7021-44c5-8399-6c21f4349e5e/crucible/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.890359257Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"datafile"} {"msg":"snapshot is oxp_4a0f3222-7021-44c5-8399-6c21f4349e5e/crucible/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881@17b5f014-6b60-49c6-a475-273fd6651c09","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.897995732Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_4a0f3222-7021-44c5-8399-6c21f4349e5e/crucible/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881\", \"17b5f014-6b60-49c6-a475-273fd6651c09\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.898022273Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot aae7f8da-77a0-49f9-bf95-d4e75fd1d881-17b5f014-6b60-49c6-a475-273fd6651c09 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.905547485Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"datafile"} {"msg":"applying SMF actions for region aae7f8da-77a0-49f9-bf95-d4e75fd1d881 running snapshot 17b5f014-6b60-49c6-a475-273fd6651c09 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.90617384Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.906220061Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"uri":"/crucible/0/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881/snapshots/17b5f014-6b60-49c6-a475-273fd6651c09/run","method":"POST","req_id":"f7ecb782-20c1-43ea-be17-54ff0a8b2aab","remote_addr":"[fd00:1122:3344:101::a]:49264","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25561,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.914153172Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.914238805Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.914321396Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.914332376Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.914373708Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-aae7f8da-77a0-49f9-bf95-d4e75fd1d881","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.914429469Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.914530031Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.914596813Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.914646224Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.914696645Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.922139936Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-aae7f8da-77a0-49f9-bf95-d4e75fd1d881-17b5f014-6b60-49c6-a475-273fd6651c09","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.922622157Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-aae7f8da-77a0-49f9-bf95-d4e75fd1d881-17b5f014-6b60-49c6-a475-273fd6651c09","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.927826106Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-aae7f8da-77a0-49f9-bf95-d4e75fd1d881-17b5f014-6b60-49c6-a475-273fd6651c09","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.944428277Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881/.zfs/snapshot/17b5f014-6b60-49c6-a475-273fd6651c09","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.944749694Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.944807075Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.944852567Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.944892377Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.944948149Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.949316799Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.957473376Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"worker"} {"msg":"region aae7f8da-77a0-49f9-bf95-d4e75fd1d881 running snapshot 17b5f014-6b60-49c6-a475-273fd6651c09 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:03.957485816Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"datafile"} {"msg":"path is \"/data/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:04.125667574Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"datafile"} {"msg":"dataset is oxp_4a0f3222-7021-44c5-8399-6c21f4349e5e/crucible/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:04.134897266Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"datafile"} {"msg":"snapshot is oxp_4a0f3222-7021-44c5-8399-6c21f4349e5e/crucible/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881@17b5f014-6b60-49c6-a475-273fd6651c09","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:04.144174028Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_4a0f3222-7021-44c5-8399-6c21f4349e5e/crucible/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881\", \"17b5f014-6b60-49c6-a475-273fd6651c09\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:04.144205999Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-16T20:00:04.152336566Z","hostname":"oxz_crucible_e02391f3-b980-4a74-b17d-9ec43c251c57","pid":20886,"uri":"/crucible/0/regions/aae7f8da-77a0-49f9-bf95-d4e75fd1d881/snapshots/17b5f014-6b60-49c6-a475-273fd6651c09/run","method":"POST","req_id":"d5eabaa5-4629-4634-8557-dcb6b0a5b44f","remote_addr":"[fd00:1122:3344:101::a]:49264","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26699,"response_code":"200"}