{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.239565086Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53498"} {"msg":"region 6c6b6641-5f49-44d5-999d-b60e4d257f59 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.239705478Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.240421757Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.240433308Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"uri":"/crucible/0/regions","method":"POST","req_id":"90faacf7-254d-4580-98a2-84c3876eb062","remote_addr":"[fd00:1122:3344:101::a]:53498","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":754,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/6c6b6641-5f49-44d5-999d-b60e4d257f59","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.253309567Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/6c6b6641-5f49-44d5-999d-b60e4d257f59","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.253332578Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"6c6b6641-5f49-44d5-999d-b60e4d257f59\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6c6b6641-5f49-44d5-999d-b60e4d257f59\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.311449984Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"region":"6c6b6641-5f49-44d5-999d-b60e4d257f59","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6c6b6641-5f49-44d5-999d-b60e4d257f59\" \"--data\" \"/data/regions/6c6b6641-5f49-44d5-999d-b60e4d257f59\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.311489854Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"region":"6c6b6641-5f49-44d5-999d-b60e4d257f59","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.43102505Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"region":"6c6b6641-5f49-44d5-999d-b60e4d257f59","component":"worker"} {"msg":"region 6c6b6641-5f49-44d5-999d-b60e4d257f59 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.431087681Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.431256133Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-6c6b6641-5f49-44d5-999d-b60e4d257f59","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.432785164Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.448344499Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"uri":"/crucible/0/regions","method":"POST","req_id":"ae8b16e2-61e2-461a-b3cb-42b12557c94a","remote_addr":"[fd00:1122:3344:101::a]:53498","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":108,"response_code":"200"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6c6b6641-5f49-44d5-999d-b60e4d257f59","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.45451574Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.480464902Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c6b6641-5f49-44d5-999d-b60e4d257f59","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.483665524Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c6b6641-5f49-44d5-999d-b60e4d257f59","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.484013829Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.48407271Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.484120401Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.484169241Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.492799865Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:35.50000838Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.214852517Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"uri":"/crucible/0/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b","method":"GET","req_id":"46ecb92f-6cfa-4270-b002-5aaae19a0cba","remote_addr":"[fd00:1122:3344:101::a]:53498","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":72,"response_code":"200"} {"msg":"path is \"/data/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.220226328Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"datafile"} {"msg":"dataset is oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.234226442Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"datafile"} {"msg":"snapshot is oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b@448d8af7-b4b4-4267-9bf7-6dced3bf96e6","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.24619262Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b\", \"448d8af7-b4b4-4267-9bf7-6dced3bf96e6\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.246225801Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.261961558Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"uri":"/crucible/0/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b/snapshots/448d8af7-b4b4-4267-9bf7-6dced3bf96e6","method":"GET","req_id":"52a97ac4-676e-4515-8b27-e25fe3e0cf56","remote_addr":"[fd00:1122:3344:101::a]:53498","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":41742,"response_code":"200"} {"msg":"path is \"/data/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.266027832Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"datafile"} {"msg":"dataset is oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.279036823Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"datafile"} {"msg":"snapshot is oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b@448d8af7-b4b4-4267-9bf7-6dced3bf96e6","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.29093339Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b\", \"448d8af7-b4b4-4267-9bf7-6dced3bf96e6\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.291000651Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot d58efed7-deb7-43e3-91f6-adbe5bc5800b-448d8af7-b4b4-4267-9bf7-6dced3bf96e6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.303253472Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"datafile"} {"msg":"applying SMF actions for region d58efed7-deb7-43e3-91f6-adbe5bc5800b running snapshot 448d8af7-b4b4-4267-9bf7-6dced3bf96e6 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.303480986Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.303508336Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"uri":"/crucible/0/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b/snapshots/448d8af7-b4b4-4267-9bf7-6dced3bf96e6/run","method":"POST","req_id":"f35cdd64-74c4-4290-9de2-e59a185012c5","remote_addr":"[fd00:1122:3344:101::a]:53498","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":37503,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-d58efed7-deb7-43e3-91f6-adbe5bc5800b-448d8af7-b4b4-4267-9bf7-6dced3bf96e6","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.307795882Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-d58efed7-deb7-43e3-91f6-adbe5bc5800b-448d8af7-b4b4-4267-9bf7-6dced3bf96e6","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.311800555Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-d58efed7-deb7-43e3-91f6-adbe5bc5800b-448d8af7-b4b4-4267-9bf7-6dced3bf96e6","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.324642145Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b/.zfs/snapshot/448d8af7-b4b4-4267-9bf7-6dced3bf96e6","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.32502085Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.325114641Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.325180092Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.325248442Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.325331283Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.329532859Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.335311325Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"worker"} {"msg":"region d58efed7-deb7-43e3-91f6-adbe5bc5800b running snapshot 448d8af7-b4b4-4267-9bf7-6dced3bf96e6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.335331415Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"datafile"} {"msg":"path is \"/data/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.547805656Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"datafile"} {"msg":"dataset is oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.557445155Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"datafile"} {"msg":"snapshot is oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b@448d8af7-b4b4-4267-9bf7-6dced3bf96e6","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.566307791Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_4e731089-319a-4a20-850f-ec8cd59d4a62/crucible/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b\", \"448d8af7-b4b4-4267-9bf7-6dced3bf96e6\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.566333191Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-17T11:37:37.575054036Z","hostname":"oxz_crucible_f4f8a43f-8c05-490c-8ca6-9e4da7ce3a9b","pid":22646,"uri":"/crucible/0/regions/d58efed7-deb7-43e3-91f6-adbe5bc5800b/snapshots/448d8af7-b4b4-4267-9bf7-6dced3bf96e6/run","method":"POST","req_id":"43c10422-4d18-42c4-817f-e02035aa2a03","remote_addr":"[fd00:1122:3344:101::a]:53498","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27268,"response_code":"200"}