{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.51208831Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:53730"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.512206752Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"uri":"/crucible/0/regions/02b93155-5699-4694-bfab-30aa677a0440","method":"GET","req_id":"c3be1b8c-91e7-48b1-8e9c-3d1f7bc708c0","remote_addr":"[fd00:1122:3344:101::b]:53730","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":50,"response_code":"200"} {"msg":"path is \"/data/regions/02b93155-5699-4694-bfab-30aa677a0440\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.515861912Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"datafile"} {"msg":"dataset is oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible/regions/02b93155-5699-4694-bfab-30aa677a0440","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.525058729Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"datafile"} {"msg":"snapshot is oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible/regions/02b93155-5699-4694-bfab-30aa677a0440@09fe4561-5243-4a46-82c8-f00e8d5fc32b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.533347138Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible/regions/02b93155-5699-4694-bfab-30aa677a0440\", \"09fe4561-5243-4a46-82c8-f00e8d5fc32b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.533378288Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.544624444Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"uri":"/crucible/0/regions/02b93155-5699-4694-bfab-30aa677a0440/snapshots/09fe4561-5243-4a46-82c8-f00e8d5fc32b","method":"GET","req_id":"c7262fa5-4fe3-4a14-8696-57dac838fde2","remote_addr":"[fd00:1122:3344:101::b]:53730","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28781,"response_code":"200"} {"msg":"path is \"/data/regions/02b93155-5699-4694-bfab-30aa677a0440\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.548862706Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"datafile"} {"msg":"dataset is oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible/regions/02b93155-5699-4694-bfab-30aa677a0440","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.557770227Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"datafile"} {"msg":"snapshot is oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible/regions/02b93155-5699-4694-bfab-30aa677a0440@09fe4561-5243-4a46-82c8-f00e8d5fc32b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.565509116Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible/regions/02b93155-5699-4694-bfab-30aa677a0440\", \"09fe4561-5243-4a46-82c8-f00e8d5fc32b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.565535666Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 02b93155-5699-4694-bfab-30aa677a0440-09fe4561-5243-4a46-82c8-f00e8d5fc32b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.574321645Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"datafile"} {"msg":"applying SMF actions for region 02b93155-5699-4694-bfab-30aa677a0440 running snapshot 09fe4561-5243-4a46-82c8-f00e8d5fc32b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.575190382Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.575202662Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"uri":"/crucible/0/regions/02b93155-5699-4694-bfab-30aa677a0440/snapshots/09fe4561-5243-4a46-82c8-f00e8d5fc32b/run","method":"POST","req_id":"cef89baf-20f4-4e3e-8e25-da81b8430ba0","remote_addr":"[fd00:1122:3344:101::b]:53730","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":26865,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/02b93155-5699-4694-bfab-30aa677a0440","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.579570416Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.579654117Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.579733229Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.579740669Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.57977936Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-02b93155-5699-4694-bfab-30aa677a0440","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.5798269Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/02b93155-5699-4694-bfab-30aa677a0440","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.579924082Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.579982914Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.580028435Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.580075875Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.589925245Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-02b93155-5699-4694-bfab-30aa677a0440-09fe4561-5243-4a46-82c8-f00e8d5fc32b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.590459915Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-02b93155-5699-4694-bfab-30aa677a0440-09fe4561-5243-4a46-82c8-f00e8d5fc32b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.596285087Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-02b93155-5699-4694-bfab-30aa677a0440-09fe4561-5243-4a46-82c8-f00e8d5fc32b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.616077777Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/02b93155-5699-4694-bfab-30aa677a0440/.zfs/snapshot/09fe4561-5243-4a46-82c8-f00e8d5fc32b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.616389493Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.616457634Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.616502355Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.616535465Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.616593767Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.623057101Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.630464983Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"worker"} {"msg":"region 02b93155-5699-4694-bfab-30aa677a0440 running snapshot 09fe4561-5243-4a46-82c8-f00e8d5fc32b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.630473413Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"datafile"} {"msg":"path is \"/data/regions/02b93155-5699-4694-bfab-30aa677a0440\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.94327896Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"datafile"} {"msg":"dataset is oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible/regions/02b93155-5699-4694-bfab-30aa677a0440","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.952082659Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"datafile"} {"msg":"snapshot is oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible/regions/02b93155-5699-4694-bfab-30aa677a0440@09fe4561-5243-4a46-82c8-f00e8d5fc32b","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.959813057Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible/regions/02b93155-5699-4694-bfab-30aa677a0440\", \"09fe4561-5243-4a46-82c8-f00e8d5fc32b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.959835938Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-16T23:36:10.96720395Z","hostname":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","pid":23464,"uri":"/crucible/0/regions/02b93155-5699-4694-bfab-30aa677a0440/snapshots/09fe4561-5243-4a46-82c8-f00e8d5fc32b/run","method":"POST","req_id":"7a1d729b-b91f-4aeb-9168-6aac0a2753af","remote_addr":"[fd00:1122:3344:101::b]:53730","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23944,"response_code":"200"}