{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.587294131Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:56370"} {"msg":"region f76a1a6d-d87d-4d36-947e-a52364411ca3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.587405832Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.591723811Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.591738951Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"uri":"/crucible/0/regions","method":"POST","req_id":"f7e2c586-c82b-466b-9d21-0409a8be4591","remote_addr":"[fd00:1122:3344:101::b]:56370","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":4369,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_c35af16a-5822-4db4-9fe5-41a043320b6e/crucible/regions/f76a1a6d-d87d-4d36-947e-a52364411ca3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.601201518Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_c35af16a-5822-4db4-9fe5-41a043320b6e/crucible/regions/f76a1a6d-d87d-4d36-947e-a52364411ca3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.601220718Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f76a1a6d-d87d-4d36-947e-a52364411ca3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f76a1a6d-d87d-4d36-947e-a52364411ca3\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.650328911Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"region":"f76a1a6d-d87d-4d36-947e-a52364411ca3","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f76a1a6d-d87d-4d36-947e-a52364411ca3\" \"--data\" \"/data/regions/f76a1a6d-d87d-4d36-947e-a52364411ca3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.650359765Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"region":"f76a1a6d-d87d-4d36-947e-a52364411ca3","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.696073669Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"region":"f76a1a6d-d87d-4d36-947e-a52364411ca3","component":"worker"} {"msg":"region f76a1a6d-d87d-4d36-947e-a52364411ca3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.696106459Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.69619314Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f76a1a6d-d87d-4d36-947e-a52364411ca3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.696975357Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f76a1a6d-d87d-4d36-947e-a52364411ca3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.70058475Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.712296857Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f76a1a6d-d87d-4d36-947e-a52364411ca3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.713514789Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f76a1a6d-d87d-4d36-947e-a52364411ca3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.713786571Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.713821431Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.713871712Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.713910232Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.715579596Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.718342853Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:17.788295045Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"uri":"/crucible/0/regions","method":"POST","req_id":"69de498c-5fae-49fa-9bc7-6bd672a55c5c","remote_addr":"[fd00:1122:3344:101::b]:56370","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":59,"response_code":200} {"msg":"region 7bb52d89-c681-4b51-8d6b-848e2d58a3ab state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.455141189Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.455334211Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"uri":"/crucible/0/regions","method":"POST","req_id":"e2e73f58-663b-4bcc-bf6e-dc3550ecb980","remote_addr":"[fd00:1122:3344:101::b]:56370","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":242,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.455372741Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_c35af16a-5822-4db4-9fe5-41a043320b6e/crucible/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.461346866Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_c35af16a-5822-4db4-9fe5-41a043320b6e/crucible/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.461371106Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"7bb52d89-c681-4b51-8d6b-848e2d58a3ab\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.496211316Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"region":"7bb52d89-c681-4b51-8d6b-848e2d58a3ab","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7bb52d89-c681-4b51-8d6b-848e2d58a3ab\" \"--data\" \"/data/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.496246896Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"region":"7bb52d89-c681-4b51-8d6b-848e2d58a3ab","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.540920606Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"region":"7bb52d89-c681-4b51-8d6b-848e2d58a3ab","component":"worker"} {"msg":"region 7bb52d89-c681-4b51-8d6b-848e2d58a3ab state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.540958267Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.541096348Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-7bb52d89-c681-4b51-8d6b-848e2d58a3ab","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.542035126Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7bb52d89-c681-4b51-8d6b-848e2d58a3ab","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.546411806Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.549567386Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7bb52d89-c681-4b51-8d6b-848e2d58a3ab","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.550963288Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.551253281Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.551296131Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.551337642Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.551372262Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.552999487Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.55658965Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:23.675317629Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"uri":"/crucible/0/regions","method":"POST","req_id":"4c301b95-4db8-4328-b787-873f736c887a","remote_addr":"[fd00:1122:3344:101::b]:56370","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":59,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.005953726Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"uri":"/crucible/0/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab","method":"GET","req_id":"5434eb9c-7bf5-4169-9f20-4a688978a897","remote_addr":"[fd00:1122:3344:101::b]:56370","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":54,"response_code":200} {"msg":"path is \"/data/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.01296142Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"datafile"} {"msg":"dataset is oxp_c35af16a-5822-4db4-9fe5-41a043320b6e/crucible/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.022540479Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"datafile"} {"msg":"region 7bb52d89-c681-4b51-8d6b-848e2d58a3ab state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.029516883Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.029674814Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.029682314Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"uri":"/crucible/0/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab","method":"DELETE","req_id":"ae43d68b-744c-4745-a6a5-971a52b1f346","remote_addr":"[fd00:1122:3344:101::b]:56370","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":16762,"response_code":204} {"msg":"disabling downstairs instance: downstairs-7bb52d89-c681-4b51-8d6b-848e2d58a3ab (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.031359989Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.034229126Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"uri":"/crucible/0/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab","method":"GET","req_id":"27f78db1-a9e2-46e6-8d39-bb879a6d4c21","remote_addr":"[fd00:1122:3344:101::b]:56370","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.035639169Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_c35af16a-5822-4db4-9fe5-41a043320b6e/crucible/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.044029606Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"region":"7bb52d89-c681-4b51-8d6b-848e2d58a3ab","component":"worker"} {"msg":"region 7bb52d89-c681-4b51-8d6b-848e2d58a3ab state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.088269392Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-04T03:03:49.116089327Z","hostname":"oxz_crucible_b250d933-ac7e-4c23-af59-8270c6e99055","pid":22092,"uri":"/crucible/0/regions/7bb52d89-c681-4b51-8d6b-848e2d58a3ab","method":"GET","req_id":"f56c668e-9d5a-461a-bab8-f418f4344ecf","remote_addr":"[fd00:1122:3344:101::b]:56370","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":56,"response_code":200}