{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.305806025Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:58756"} {"msg":"region f802a145-34de-4cf9-9a17-a1ee5a5254e7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.305901336Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.309773158Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.309802728Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions","method":"POST","req_id":"347f678f-30d2-4ab4-9ebd-d2faca8cbf9a","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":3916,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/f802a145-34de-4cf9-9a17-a1ee5a5254e7","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.31903688Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/f802a145-34de-4cf9-9a17-a1ee5a5254e7","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.31905965Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f802a145-34de-4cf9-9a17-a1ee5a5254e7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f802a145-34de-4cf9-9a17-a1ee5a5254e7\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.364810732Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"region":"f802a145-34de-4cf9-9a17-a1ee5a5254e7","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f802a145-34de-4cf9-9a17-a1ee5a5254e7\" \"--data\" \"/data/regions/f802a145-34de-4cf9-9a17-a1ee5a5254e7\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.364842552Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"region":"f802a145-34de-4cf9-9a17-a1ee5a5254e7","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.409143519Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"region":"f802a145-34de-4cf9-9a17-a1ee5a5254e7","component":"worker"} {"msg":"region f802a145-34de-4cf9-9a17-a1ee5a5254e7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.409177709Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.40928596Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.423101212Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.423175383Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.423252504Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.423267624Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.423299834Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.423352114Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.423445675Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.423501396Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.423549117Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.423594807Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.427752723Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f802a145-34de-4cf9-9a17-a1ee5a5254e7","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.428143017Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f802a145-34de-4cf9-9a17-a1ee5a5254e7","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.429164658Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.435302516Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f802a145-34de-4cf9-9a17-a1ee5a5254e7","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.436420378Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f802a145-34de-4cf9-9a17-a1ee5a5254e7","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.436724791Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.436781492Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.436818972Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.436855583Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.438729273Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.442228232Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:52.461499873Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions","method":"POST","req_id":"a7ede4a6-5102-431e-aff3-3eeacb14d5b1","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":57,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.456274368Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","method":"GET","req_id":"4cc6b5a1-dc93-4b94-9b5d-3942b10577e7","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":70,"response_code":200} {"msg":"path is \"/data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.459301301Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"dataset is oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.464781771Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"snapshot is oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034@50122ef4-5a8d-4104-8b55-674aa5d9dc0f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.469828136Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034\", \"50122ef4-5a8d-4104-8b55-674aa5d9dc0f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.469849907Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.476780223Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034/snapshots/50122ef4-5a8d-4104-8b55-674aa5d9dc0f","method":"GET","req_id":"d0a79444-3d20-49dc-ae5a-6dedfabf72a0","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":17512,"response_code":200} {"msg":"path is \"/data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.480448003Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"dataset is oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.487240997Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"snapshot is oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034@50122ef4-5a8d-4104-8b55-674aa5d9dc0f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.493596537Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034\", \"50122ef4-5a8d-4104-8b55-674aa5d9dc0f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.493617997Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot a30a645b-e6f3-4a59-b7d2-ab4cd8a67034-50122ef4-5a8d-4104-8b55-674aa5d9dc0f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.500635014Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"applying SMF actions for region a30a645b-e6f3-4a59-b7d2-ab4cd8a67034 running snapshot 50122ef4-5a8d-4104-8b55-674aa5d9dc0f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.500789986Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.500803386Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034/snapshots/50122ef4-5a8d-4104-8b55-674aa5d9dc0f/run","method":"POST","req_id":"db7ad4ac-8dee-47b9-8a50-6c7264c16ae0","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20401,"response_code":200} {"msg":"existing directory value /dev/null does not match /data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.501302342Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.501375182Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.501447093Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.501456873Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.501491004Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.501531674Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.501884618Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.501932399Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.501975449Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.50201602Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.503996201Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-a30a645b-e6f3-4a59-b7d2-ab4cd8a67034-50122ef4-5a8d-4104-8b55-674aa5d9dc0f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.507974575Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-a30a645b-e6f3-4a59-b7d2-ab4cd8a67034-50122ef4-5a8d-4104-8b55-674aa5d9dc0f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.517460819Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-a30a645b-e6f3-4a59-b7d2-ab4cd8a67034-50122ef4-5a8d-4104-8b55-674aa5d9dc0f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.522265602Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034/.zfs/snapshot/50122ef4-5a8d-4104-8b55-674aa5d9dc0f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.522528605Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.522583965Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.522619846Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.522656096Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.522707647Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.524666838Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.527347098Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"region a30a645b-e6f3-4a59-b7d2-ab4cd8a67034 running snapshot 50122ef4-5a8d-4104-8b55-674aa5d9dc0f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.527357667Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"path is \"/data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.748249651Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"dataset is oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.754387129Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"snapshot is oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034@50122ef4-5a8d-4104-8b55-674aa5d9dc0f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.760708998Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034\", \"50122ef4-5a8d-4104-8b55-674aa5d9dc0f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.760749528Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:53.766255189Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034/snapshots/50122ef4-5a8d-4104-8b55-674aa5d9dc0f/run","method":"POST","req_id":"96cc3098-311e-47e2-9556-56f1a89f949c","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18042,"response_code":200} {"msg":"region 545f6710-cc9a-4021-b5fc-e7bf6060220a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.362780973Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.362949425Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions","method":"POST","req_id":"3b90a63d-fa75-4e83-824a-436f87de365b","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":242,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.362977135Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.369501587Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.369538937Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"545f6710-cc9a-4021-b5fc-e7bf6060220a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.41851633Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"region":"545f6710-cc9a-4021-b5fc-e7bf6060220a","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"545f6710-cc9a-4021-b5fc-e7bf6060220a\" \"--data\" \"/data/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.418620755Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"region":"545f6710-cc9a-4021-b5fc-e7bf6060220a","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.46736478Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"region":"545f6710-cc9a-4021-b5fc-e7bf6060220a","component":"worker"} {"msg":"region 545f6710-cc9a-4021-b5fc-e7bf6060220a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.467400911Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.467512312Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-545f6710-cc9a-4021-b5fc-e7bf6060220a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.468029557Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-545f6710-cc9a-4021-b5fc-e7bf6060220a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.469489664Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.472955402Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-545f6710-cc9a-4021-b5fc-e7bf6060220a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.474171745Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.474438258Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.474483368Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.474527809Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.474563229Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.476271639Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.479428443Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.479530304Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.479616105Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.479636255Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.479652185Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.479701196Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.482263574Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.482314334Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.482358555Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.482400025Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.484297686Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.486858828Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:05:58.615367314Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions","method":"POST","req_id":"0458e85a-949c-49a2-8a84-7fcbecf17ef6","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":63,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.709661699Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a","method":"GET","req_id":"974cdd4e-9149-4146-8d58-d0cacb3062b7","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":53,"response_code":200} {"msg":"path is \"/data/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.713743743Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"dataset is oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.722025754Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"region 545f6710-cc9a-4021-b5fc-e7bf6060220a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.729094602Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.729270234Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.729281794Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a","method":"DELETE","req_id":"92a6b593-5a3d-489d-99bd-275a8ebc366e","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":15589,"response_code":204} {"msg":"disabling downstairs instance: downstairs-545f6710-cc9a-4021-b5fc-e7bf6060220a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.730319695Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.733016915Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.733128886Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.733209427Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.733217177Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.733252737Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.733299858Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a30a645b-e6f3-4a59-b7d2-ab4cd8a67034","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.733605131Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.733672102Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.733720943Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.733762983Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.734099827Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a","method":"GET","req_id":"bf65005f-19a5-46c8-aca7-df410ab13406","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":65,"response_code":200} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.736116319Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.738276042Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_0aae25c1-1445-4dd1-b058-bae7a5741994/crucible/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.747495714Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"region":"545f6710-cc9a-4021-b5fc-e7bf6060220a","component":"worker"} {"msg":"region 545f6710-cc9a-4021-b5fc-e7bf6060220a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.800511985Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-11T15:06:25.810205072Z","hostname":"oxz_crucible_2d94fdca-0efa-4749-9d58-54b72ddb37ec","pid":23424,"uri":"/crucible/0/regions/545f6710-cc9a-4021-b5fc-e7bf6060220a","method":"GET","req_id":"725784d1-1703-4584-a863-ddc768eb320a","remote_addr":"[fd00:1122:3344:101::b]:58756","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":82,"response_code":200}