{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.535026495Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54281"} {"msg":"region 6c4d1050-89ff-49f2-9fe5-abf525ccdbc5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.535094487Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.535748881Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.535761952Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"uri":"/crucible/0/regions","method":"POST","req_id":"a10ab040-e86e-4f7e-8384-2cb6fd619061","remote_addr":"[fd00:1122:3344:101::c]:54281","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":682,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/6c4d1050-89ff-49f2-9fe5-abf525ccdbc5","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.547498286Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/6c4d1050-89ff-49f2-9fe5-abf525ccdbc5","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.547537757Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"6c4d1050-89ff-49f2-9fe5-abf525ccdbc5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6c4d1050-89ff-49f2-9fe5-abf525ccdbc5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.627444368Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"region":"6c4d1050-89ff-49f2-9fe5-abf525ccdbc5","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6c4d1050-89ff-49f2-9fe5-abf525ccdbc5\" \"--data\" \"/data/regions/6c4d1050-89ff-49f2-9fe5-abf525ccdbc5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.629393512Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"region":"6c4d1050-89ff-49f2-9fe5-abf525ccdbc5","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.897370153Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"region":"6c4d1050-89ff-49f2-9fe5-abf525ccdbc5","component":"worker"} {"msg":"region 6c4d1050-89ff-49f2-9fe5-abf525ccdbc5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.897424154Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.897527797Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.908544735Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"uri":"/crucible/0/regions","method":"POST","req_id":"ef636879-d812-4877-9489-3fd9d3143401","remote_addr":"[fd00:1122:3344:101::c]:54281","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":92,"response_code":"200"} {"msg":"creating missing downstairs instance downstairs-6c4d1050-89ff-49f2-9fe5-abf525ccdbc5","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.926127012Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6c4d1050-89ff-49f2-9fe5-abf525ccdbc5","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.955259158Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.973800826Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c4d1050-89ff-49f2-9fe5-abf525ccdbc5","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.990263327Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c4d1050-89ff-49f2-9fe5-abf525ccdbc5","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.990631886Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.990700047Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.990762529Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:01.99081949Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:02.002117134Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:02.013959672Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.280284806Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"uri":"/crucible/0/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0","method":"GET","req_id":"4e2d7b58-a746-49b1-b4a8-667fd6e35c1d","remote_addr":"[fd00:1122:3344:101::c]:54281","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":47,"response_code":"200"} {"msg":"path is \"/data/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.289956004Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"datafile"} {"msg":"dataset is oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.300061562Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"datafile"} {"msg":"snapshot is oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0@fe402274-71cb-4663-b868-094fc9c911d8","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.308656265Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0\", \"fe402274-71cb-4663-b868-094fc9c911d8\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.308687546Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.326304343Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"uri":"/crucible/0/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0/snapshots/fe402274-71cb-4663-b868-094fc9c911d8","method":"GET","req_id":"31b3aa6d-7ca1-40b6-8d81-88ce17aac907","remote_addr":"[fd00:1122:3344:101::c]:54281","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":42759,"response_code":"200"} {"msg":"path is \"/data/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.329633778Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"datafile"} {"msg":"dataset is oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.338678472Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"datafile"} {"msg":"snapshot is oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0@fe402274-71cb-4663-b868-094fc9c911d8","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.34700174Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0\", \"fe402274-71cb-4663-b868-094fc9c911d8\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.34702862Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 3ca18dd8-30e8-4b2e-9b6a-6115300d18f0-fe402274-71cb-4663-b868-094fc9c911d8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.360577496Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"datafile"} {"msg":"applying SMF actions for region 3ca18dd8-30e8-4b2e-9b6a-6115300d18f0 running snapshot fe402274-71cb-4663-b868-094fc9c911d8 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.36075Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.36077379Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"uri":"/crucible/0/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0/snapshots/fe402274-71cb-4663-b868-094fc9c911d8/run","method":"POST","req_id":"e319f1bd-30a7-4151-b8cb-0d22d959fa83","remote_addr":"[fd00:1122:3344:101::c]:54281","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":31182,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-3ca18dd8-30e8-4b2e-9b6a-6115300d18f0-fe402274-71cb-4663-b868-094fc9c911d8","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.363459991Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-3ca18dd8-30e8-4b2e-9b6a-6115300d18f0-fe402274-71cb-4663-b868-094fc9c911d8","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.384844913Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-3ca18dd8-30e8-4b2e-9b6a-6115300d18f0-fe402274-71cb-4663-b868-094fc9c911d8","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.414686006Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0/.zfs/snapshot/fe402274-71cb-4663-b868-094fc9c911d8","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.415006783Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.415074194Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.415126325Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.415178607Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.415242088Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.426112913Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.437987631Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"worker"} {"msg":"region 3ca18dd8-30e8-4b2e-9b6a-6115300d18f0 running snapshot fe402274-71cb-4663-b868-094fc9c911d8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.437999481Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"datafile"} {"msg":"path is \"/data/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.732645303Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"datafile"} {"msg":"dataset is oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.741486212Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"datafile"} {"msg":"snapshot is oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0@fe402274-71cb-4663-b868-094fc9c911d8","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.749580175Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_8d813973-46ec-459a-bf10-547f8bb041f6/crucible/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0\", \"fe402274-71cb-4663-b868-094fc9c911d8\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.749606445Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-04-24T06:49:03.758082976Z","hostname":"oxz_crucible_b115c15b-0483-4110-b306-80104808f8aa","pid":23144,"uri":"/crucible/0/regions/3ca18dd8-30e8-4b2e-9b6a-6115300d18f0/snapshots/fe402274-71cb-4663-b868-094fc9c911d8/run","method":"POST","req_id":"282dae46-0dfe-44dd-b0ae-98e2c5b3e854","remote_addr":"[fd00:1122:3344:101::c]:54281","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25470,"response_code":"200"}