{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.192618041Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61054"}
{"msg":"region 8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.223716092Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.228258128Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.232436728Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions","method":"POST","req_id":"4f69fe6b-96a8-42b5-982b-e7ba37df52a6","remote_addr":"[fd00:1122:3344:101::c]:61054","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":15870,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.238556551Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.238581141Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.293889678Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac\" \"--data\" \"/data/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.29579255Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.45335888Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","component":"worker"}
{"msg":"region 8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.457345077Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.457676563Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.459973531Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.464943765Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.475494521Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.476964466Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.477698108Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.477763639Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.47780235Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.47784035Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.480736239Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.483642458Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:18.539642216Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions","method":"POST","req_id":"78df8382-baf0-460c-a018-26ddda6c5981","remote_addr":"[fd00:1122:3344:101::c]:61054","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":72,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.541568125Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","method":"GET","req_id":"d75e5c7a-79df-4890-a88f-0bfd3cccbf2d","remote_addr":"[fd00:1122:3344:101::c]:61054","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":94,"response_code":200}
{"msg":"path is \"/data/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.544780269Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"dataset is oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.552493548Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"region 8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.560307889Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.560454841Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.560485152Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","method":"DELETE","req_id":"aab392af-a73b-4bd8-a993-6660e9ca15ce","remote_addr":"[fd00:1122:3344:101::c]:61054","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":15718,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.56155734Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.563869308Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","method":"GET","req_id":"0c0517ae-0faa-4c2b-aa80-7bcb79c068f3","remote_addr":"[fd00:1122:3344:101::c]:61054","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":47,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.564472309Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.573398158Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","component":"worker"}
{"msg":"region 8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.647861566Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:19.692695727Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/8d9117e5-2fd6-4fe9-a9b7-631b9defd3ac","method":"GET","req_id":"5cc341e3-fe8c-48a8-a685-0529fe34835f","remote_addr":"[fd00:1122:3344:101::c]:61054","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":52,"response_code":200}
{"msg":"region e889f719-aba3-4536-b268-e67418cd5c86 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.033151466Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.03335709Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions","method":"POST","req_id":"a7f66d0e-1488-4b7f-a0a1-cafb570ef454","remote_addr":"[fd00:1122:3344:101::c]:61054","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":253,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.03337301Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.039504683Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.039539723Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e889f719-aba3-4536-b268-e67418cd5c86\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e889f719-aba3-4536-b268-e67418cd5c86\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.100728699Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"e889f719-aba3-4536-b268-e67418cd5c86","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e889f719-aba3-4536-b268-e67418cd5c86\" \"--data\" \"/data/regions/e889f719-aba3-4536-b268-e67418cd5c86\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.10080037Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"e889f719-aba3-4536-b268-e67418cd5c86","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.16944555Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"e889f719-aba3-4536-b268-e67418cd5c86","component":"worker"}
{"msg":"region e889f719-aba3-4536-b268-e67418cd5c86 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.169485821Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.169569592Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.169849387Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions","method":"POST","req_id":"abbebed4-5eae-491e-8204-774e81d95fe2","remote_addr":"[fd00:1122:3344:101::c]:61054","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"creating missing downstairs instance downstairs-e889f719-aba3-4536-b268-e67418cd5c86","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.170990926Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e889f719-aba3-4536-b268-e67418cd5c86","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.172166376Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.175972639Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e889f719-aba3-4536-b268-e67418cd5c86","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.17717465Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e889f719-aba3-4536-b268-e67418cd5c86","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.177411574Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.177470175Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.177512485Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.177549546Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.179336476Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:00:22.182276945Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:25.968402005Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:63992"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:25.968509032Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/e889f719-aba3-4536-b268-e67418cd5c86","method":"GET","req_id":"e045dfcd-7946-4bca-87cd-d56113e821fe","remote_addr":"[fd00:1122:3344:101::c]:63992","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":40,"response_code":200}
{"msg":"path is \"/data/regions/e889f719-aba3-4536-b268-e67418cd5c86\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:25.971485476Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"dataset is oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:25.977801722Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"snapshot is oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86@5dafbd5f-4792-4af7-a407-2bf3937fbf1e","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:25.984512064Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86\", \"5dafbd5f-4792-4af7-a407-2bf3937fbf1e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:25.984541775Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:25.996575886Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/e889f719-aba3-4536-b268-e67418cd5c86/snapshots/5dafbd5f-4792-4af7-a407-2bf3937fbf1e","method":"GET","req_id":"944eed23-6153-4ad1-ac94-1cfea64f9c07","remote_addr":"[fd00:1122:3344:101::c]:63992","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25097,"response_code":200}
{"msg":"path is \"/data/regions/e889f719-aba3-4536-b268-e67418cd5c86\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.004710423Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"dataset is oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.012080076Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"snapshot is oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86@5dafbd5f-4792-4af7-a407-2bf3937fbf1e","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.018841539Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86\", \"5dafbd5f-4792-4af7-a407-2bf3937fbf1e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.01886643Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot e889f719-aba3-4536-b268-e67418cd5c86-5dafbd5f-4792-4af7-a407-2bf3937fbf1e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.026287214Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"applying SMF actions for region e889f719-aba3-4536-b268-e67418cd5c86 running snapshot 5dafbd5f-4792-4af7-a407-2bf3937fbf1e (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.03261554Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.032648801Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/e889f719-aba3-4536-b268-e67418cd5c86/snapshots/5dafbd5f-4792-4af7-a407-2bf3937fbf1e/run","method":"POST","req_id":"84e6f61f-4446-43cf-9bfe-049ac7c33416","remote_addr":"[fd00:1122:3344:101::c]:63992","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":27951,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-e889f719-aba3-4536-b268-e67418cd5c86-5dafbd5f-4792-4af7-a407-2bf3937fbf1e","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.043107316Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-e889f719-aba3-4536-b268-e67418cd5c86-5dafbd5f-4792-4af7-a407-2bf3937fbf1e","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.053597192Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-e889f719-aba3-4536-b268-e67418cd5c86-5dafbd5f-4792-4af7-a407-2bf3937fbf1e","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.060874334Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e889f719-aba3-4536-b268-e67418cd5c86/.zfs/snapshot/5dafbd5f-4792-4af7-a407-2bf3937fbf1e","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.061170069Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.06121442Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.06125479Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.061291141Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.061336852Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.063445057Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.066283254Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"region e889f719-aba3-4536-b268-e67418cd5c86 running snapshot 5dafbd5f-4792-4af7-a407-2bf3937fbf1e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.066301015Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"path is \"/data/regions/e889f719-aba3-4536-b268-e67418cd5c86\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.218306952Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"dataset is oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.224528436Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"snapshot is oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86@5dafbd5f-4792-4af7-a407-2bf3937fbf1e","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.231587495Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/e889f719-aba3-4536-b268-e67418cd5c86\", \"5dafbd5f-4792-4af7-a407-2bf3937fbf1e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.231621235Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:26.237168978Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/e889f719-aba3-4536-b268-e67418cd5c86/snapshots/5dafbd5f-4792-4af7-a407-2bf3937fbf1e/run","method":"POST","req_id":"7ef2c658-72e6-4edb-b0dd-f706a3b6d36e","remote_addr":"[fd00:1122:3344:101::c]:63992","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":18897,"response_code":200}
{"msg":"region b40bbb18-9a64-48dc-b1ad-606f93280e54 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.787068725Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.787235987Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions","method":"POST","req_id":"338ec1e6-e4f6-4488-9d4e-028ecb714e02","remote_addr":"[fd00:1122:3344:101::c]:63992","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":223,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.787261398Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.793058735Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.793111356Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b40bbb18-9a64-48dc-b1ad-606f93280e54\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.840332717Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"b40bbb18-9a64-48dc-b1ad-606f93280e54","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b40bbb18-9a64-48dc-b1ad-606f93280e54\" \"--data\" \"/data/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.840386668Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"b40bbb18-9a64-48dc-b1ad-606f93280e54","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.888710788Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"b40bbb18-9a64-48dc-b1ad-606f93280e54","component":"worker"}
{"msg":"region b40bbb18-9a64-48dc-b1ad-606f93280e54 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.888742348Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.88884453Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b40bbb18-9a64-48dc-b1ad-606f93280e54","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.889688454Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b40bbb18-9a64-48dc-b1ad-606f93280e54","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.890810653Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.893980806Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b40bbb18-9a64-48dc-b1ad-606f93280e54","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.895329099Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.895533862Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.895567413Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.895601103Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.895634954Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.897086808Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.903650558Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:31.977935303Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions","method":"POST","req_id":"3a14be8a-9063-4002-93a4-9ed29849f92a","remote_addr":"[fd00:1122:3344:101::c]:63992","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":72,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.719289638Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54","method":"GET","req_id":"59b348e7-cdc4-45a1-b034-9727cd5f67b0","remote_addr":"[fd00:1122:3344:101::c]:63992","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"path is \"/data/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.722517162Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"dataset is oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.731810708Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"region b40bbb18-9a64-48dc-b1ad-606f93280e54 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.739592898Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.739761241Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.739782322Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54","method":"DELETE","req_id":"8298d54c-5dff-4754-b62f-d040ef658420","remote_addr":"[fd00:1122:3344:101::c]:63992","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":17263,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-b40bbb18-9a64-48dc-b1ad-606f93280e54 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.741280857Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.743458316Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54","method":"GET","req_id":"8460ff20-4c94-4ae8-a693-a31cc9bd78e9","remote_addr":"[fd00:1122:3344:101::c]:63992","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":29,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.745896084Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ecf77cbf-e224-4f9f-8bde-90c3d7ea4fb4/crucible/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.754803743Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"region":"b40bbb18-9a64-48dc-b1ad-606f93280e54","component":"worker"}
{"msg":"region b40bbb18-9a64-48dc-b1ad-606f93280e54 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.80836746Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-29T03:11:57.833732983Z","hostname":"oxz_crucible_1b43a057-0213-48fe-9db0-100ec2715890","pid":22381,"uri":"/crucible/0/regions/b40bbb18-9a64-48dc-b1ad-606f93280e54","method":"GET","req_id":"6ce99727-e3a9-444e-b912-1a72a9de4f5c","remote_addr":"[fd00:1122:3344:101::c]:63992","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":54,"response_code":200}