{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.253361727Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62780"} {"msg":"region e1c591a2-e77b-4550-9466-5dd2bf95c190 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.253456268Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.254049624Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"uri":"/crucible/0/regions","method":"POST","req_id":"abeb6fdd-d7e3-47a9-a302-a0d7b2c8b4f4","remote_addr":"[fd00:1122:3344:101::a]:62780","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":632,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.254064345Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/e1c591a2-e77b-4550-9466-5dd2bf95c190","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.268164271Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/e1c591a2-e77b-4550-9466-5dd2bf95c190","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.268189251Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e1c591a2-e77b-4550-9466-5dd2bf95c190\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e1c591a2-e77b-4550-9466-5dd2bf95c190\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.333981958Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"region":"e1c591a2-e77b-4550-9466-5dd2bf95c190","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e1c591a2-e77b-4550-9466-5dd2bf95c190\" \"--data\" \"/data/regions/e1c591a2-e77b-4550-9466-5dd2bf95c190\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.334031558Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"region":"e1c591a2-e77b-4550-9466-5dd2bf95c190","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.417620622Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"region":"e1c591a2-e77b-4550-9466-5dd2bf95c190","component":"worker"} {"msg":"region e1c591a2-e77b-4550-9466-5dd2bf95c190 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.417690863Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.417962616Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e1c591a2-e77b-4550-9466-5dd2bf95c190","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.419388472Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e1c591a2-e77b-4550-9466-5dd2bf95c190","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.442242374Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.477658956Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e1c591a2-e77b-4550-9466-5dd2bf95c190","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.484022236Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e1c591a2-e77b-4550-9466-5dd2bf95c190","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.48436618Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.48442287Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.484465911Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.484515202Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.491130255Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.497789498Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:02.599724445Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"uri":"/crucible/0/regions","method":"POST","req_id":"b9c6ae38-d799-4619-871d-d2d6a209a806","remote_addr":"[fd00:1122:3344:101::a]:62780","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":83,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.07625116Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"uri":"/crucible/0/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4","method":"GET","req_id":"549c0586-6511-4f11-bc3b-24b2039614d3","remote_addr":"[fd00:1122:3344:101::a]:62780","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":48,"response_code":200} {"msg":"path is \"/data/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.080228534Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"datafile"} {"msg":"dataset is oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.089365445Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"datafile"} {"msg":"snapshot is oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4@c31b42a6-2942-4d80-aa41-3218c533595f","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.097292883Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4\", \"c31b42a6-2942-4d80-aa41-3218c533595f\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.097318093Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.107961251Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"uri":"/crucible/0/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4/snapshots/c31b42a6-2942-4d80-aa41-3218c533595f","method":"GET","req_id":"9bafd6e6-832d-4c7e-b44a-fd9ba41937de","remote_addr":"[fd00:1122:3344:101::a]:62780","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27735,"response_code":200} {"msg":"path is \"/data/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.111372678Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"datafile"} {"msg":"dataset is oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.118908642Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"datafile"} {"msg":"snapshot is oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4@c31b42a6-2942-4d80-aa41-3218c533595f","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.126525926Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4\", \"c31b42a6-2942-4d80-aa41-3218c533595f\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.126552696Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot eb4f1458-9cfc-48a2-bf54-d4c984807fa4-c31b42a6-2942-4d80-aa41-3218c533595f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.13594972Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"datafile"} {"msg":"applying SMF actions for region eb4f1458-9cfc-48a2-bf54-d4c984807fa4 running snapshot c31b42a6-2942-4d80-aa41-3218c533595f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.136151942Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.136165212Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"uri":"/crucible/0/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4/snapshots/c31b42a6-2942-4d80-aa41-3218c533595f/run","method":"POST","req_id":"ed7551de-a422-4663-87f1-76ab4944e165","remote_addr":"[fd00:1122:3344:101::a]:62780","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":24817,"response_code":200} {"msg":"creating missing snapshot instance snapshot-eb4f1458-9cfc-48a2-bf54-d4c984807fa4-c31b42a6-2942-4d80-aa41-3218c533595f","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.138811342Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-eb4f1458-9cfc-48a2-bf54-d4c984807fa4-c31b42a6-2942-4d80-aa41-3218c533595f","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.141882686Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-eb4f1458-9cfc-48a2-bf54-d4c984807fa4-c31b42a6-2942-4d80-aa41-3218c533595f","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.152874707Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4/.zfs/snapshot/c31b42a6-2942-4d80-aa41-3218c533595f","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.153220971Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.153283682Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.153329262Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.153372423Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.153435473Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.157170945Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.161772715Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"worker"} {"msg":"region eb4f1458-9cfc-48a2-bf54-d4c984807fa4 running snapshot c31b42a6-2942-4d80-aa41-3218c533595f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.161784436Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"datafile"} {"msg":"path is \"/data/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.344647607Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"datafile"} {"msg":"dataset is oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.354106611Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"datafile"} {"msg":"snapshot is oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4@c31b42a6-2942-4d80-aa41-3218c533595f","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.363357623Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_1c2add25-73a7-4293-b69b-c31c9e5ca4b5/crucible/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4\", \"c31b42a6-2942-4d80-aa41-3218c533595f\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.363391954Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-28T22:03:03.371266141Z","hostname":"oxz_crucible_f8585368-de2d-499e-a5e6-b5ea1f148237","pid":23575,"uri":"/crucible/0/regions/eb4f1458-9cfc-48a2-bf54-d4c984807fa4/snapshots/c31b42a6-2942-4d80-aa41-3218c533595f/run","method":"POST","req_id":"0c1ca4d3-1f32-4cc5-922e-91926ae14130","remote_addr":"[fd00:1122:3344:101::a]:62780","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":26642,"response_code":200}