{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.17493529Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60848"} {"msg":"region 958a29db-8a60-494e-a58a-e9c7c9932d1a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.175036162Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.175682713Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"uri":"/crucible/0/regions","method":"POST","req_id":"19ec4198-e8a6-4eb0-8f17-df60007aa4e0","remote_addr":"[fd00:1122:3344:101::a]:60848","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":690,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.175696243Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_a076fc12-70d4-449f-97a9-480c2623f481/crucible/regions/958a29db-8a60-494e-a58a-e9c7c9932d1a","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.182542641Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_a076fc12-70d4-449f-97a9-480c2623f481/crucible/regions/958a29db-8a60-494e-a58a-e9c7c9932d1a","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.182557331Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"958a29db-8a60-494e-a58a-e9c7c9932d1a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/958a29db-8a60-494e-a58a-e9c7c9932d1a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.228438092Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"region":"958a29db-8a60-494e-a58a-e9c7c9932d1a","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"958a29db-8a60-494e-a58a-e9c7c9932d1a\" \"--data\" \"/data/regions/958a29db-8a60-494e-a58a-e9c7c9932d1a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.228465882Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"region":"958a29db-8a60-494e-a58a-e9c7c9932d1a","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.30783903Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"region":"958a29db-8a60-494e-a58a-e9c7c9932d1a","component":"worker"} {"msg":"region 958a29db-8a60-494e-a58a-e9c7c9932d1a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.307874981Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.307964982Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-958a29db-8a60-494e-a58a-e9c7c9932d1a","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.309159473Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-958a29db-8a60-494e-a58a-e9c7c9932d1a","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.318370411Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.340025855Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-958a29db-8a60-494e-a58a-e9c7c9932d1a","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.342974826Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/958a29db-8a60-494e-a58a-e9c7c9932d1a","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.34321047Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.34325189Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.343287861Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.343322832Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.346589248Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.350932123Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:37.421450848Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"uri":"/crucible/0/regions","method":"POST","req_id":"ca8c176a-6e49-411a-b0b5-27c3f224014c","remote_addr":"[fd00:1122:3344:101::a]:60848","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"region ea0b206d-f938-482d-8e91-c51c4bfa58df state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.055079112Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.055917706Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"uri":"/crucible/0/regions","method":"POST","req_id":"1fe07e70-c442-40b3-815b-74f31e034dd0","remote_addr":"[fd00:1122:3344:101::a]:60848","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":255,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.055950317Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_a076fc12-70d4-449f-97a9-480c2623f481/crucible/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.062000421Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_a076fc12-70d4-449f-97a9-480c2623f481/crucible/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.062016691Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"ea0b206d-f938-482d-8e91-c51c4bfa58df\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.119473001Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"region":"ea0b206d-f938-482d-8e91-c51c4bfa58df","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ea0b206d-f938-482d-8e91-c51c4bfa58df\" \"--data\" \"/data/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.119503242Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"region":"ea0b206d-f938-482d-8e91-c51c4bfa58df","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.192986958Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"region":"ea0b206d-f938-482d-8e91-c51c4bfa58df","component":"worker"} {"msg":"region ea0b206d-f938-482d-8e91-c51c4bfa58df state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.193021279Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.19311475Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-ea0b206d-f938-482d-8e91-c51c4bfa58df","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.197864192Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ea0b206d-f938-482d-8e91-c51c4bfa58df","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.207753563Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.216951181Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ea0b206d-f938-482d-8e91-c51c4bfa58df","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.220095795Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.220457012Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.220523043Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.220574283Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.220622864Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.225294735Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.230450034Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:00:44.338623998Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"uri":"/crucible/0/regions","method":"POST","req_id":"d9c45a1a-74a2-4d3d-86d6-5e761891c333","remote_addr":"[fd00:1122:3344:101::a]:60848","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":70,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.949385165Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59672"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.949522847Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"uri":"/crucible/0/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df","method":"GET","req_id":"fadc0bd5-be9c-48a4-af4b-27bfc0974e44","remote_addr":"[fd00:1122:3344:101::a]:59672","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":39,"response_code":200} {"msg":"path is \"/data/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.95257015Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"datafile"} {"msg":"dataset is oxp_a076fc12-70d4-449f-97a9-480c2623f481/crucible/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.964626077Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"datafile"} {"msg":"region ea0b206d-f938-482d-8e91-c51c4bfa58df state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.971580808Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.97173824Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"uri":"/crucible/0/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df","method":"DELETE","req_id":"a207d304-16cb-42eb-ac9f-01418cf523c8","remote_addr":"[fd00:1122:3344:101::a]:59672","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19194,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.97175986Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-ea0b206d-f938-482d-8e91-c51c4bfa58df (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.973212416Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.975139549Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"uri":"/crucible/0/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df","method":"GET","req_id":"7557a684-6ccf-4556-8b62-149f33efd682","remote_addr":"[fd00:1122:3344:101::a]:59672","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":67,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.989177241Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_a076fc12-70d4-449f-97a9-480c2623f481/crucible/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:18.996572418Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"region":"ea0b206d-f938-482d-8e91-c51c4bfa58df","component":"worker"} {"msg":"region ea0b206d-f938-482d-8e91-c51c4bfa58df state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:19.042493049Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-11T04:01:19.116023126Z","hostname":"oxz_crucible_7a6b91a1-53fd-485d-ab35-e767436815b8","pid":24670,"uri":"/crucible/0/regions/ea0b206d-f938-482d-8e91-c51c4bfa58df","method":"GET","req_id":"3f3e2b78-e1e1-4ab3-8e71-ec422d9cfbf5","remote_addr":"[fd00:1122:3344:101::a]:59672","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":47,"response_code":200}