{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:56.562535929Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:57053"} {"msg":"region cec725e0-4a27-442c-95b2-e48ad2582849 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:56.612498262Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:56.615346365Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"uri":"/crucible/0/regions","method":"POST","req_id":"d4016d33-22bc-4e26-9ce4-dd38b84e4cad","remote_addr":"[fd00:1122:3344:101::c]:57053","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":20380,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:56.615411177Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_e4d000b9-898b-4a52-af6d-9050ee76dbb8/crucible/regions/cec725e0-4a27-442c-95b2-e48ad2582849","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:56.626545002Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_e4d000b9-898b-4a52-af6d-9050ee76dbb8/crucible/regions/cec725e0-4a27-442c-95b2-e48ad2582849","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:56.626563033Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"cec725e0-4a27-442c-95b2-e48ad2582849\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cec725e0-4a27-442c-95b2-e48ad2582849\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:56.677952493Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"region":"cec725e0-4a27-442c-95b2-e48ad2582849","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cec725e0-4a27-442c-95b2-e48ad2582849\" \"--data\" \"/data/regions/cec725e0-4a27-442c-95b2-e48ad2582849\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:56.680337364Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"region":"cec725e0-4a27-442c-95b2-e48ad2582849","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:56.902472309Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"uri":"/crucible/0/regions","method":"POST","req_id":"0a75bdd0-e09b-402b-9956-757c765cd4b9","remote_addr":"[fd00:1122:3344:101::c]:57053","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":63,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.034104159Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"region":"cec725e0-4a27-442c-95b2-e48ad2582849","component":"worker"} {"msg":"region cec725e0-4a27-442c-95b2-e48ad2582849 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.03414961Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.034239412Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-cec725e0-4a27-442c-95b2-e48ad2582849","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.034574531Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cec725e0-4a27-442c-95b2-e48ad2582849","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.047544074Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.069779455Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cec725e0-4a27-442c-95b2-e48ad2582849","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.072550626Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cec725e0-4a27-442c-95b2-e48ad2582849","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.075683327Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.075752609Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.07580148Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.075854771Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.079146926Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.094415808Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:54:57.516263671Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"uri":"/crucible/0/regions","method":"POST","req_id":"5e3bab7e-dbb4-4244-8913-f27aca1b4eaa","remote_addr":"[fd00:1122:3344:101::c]:57053","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":59,"response_code":200} {"msg":"region 22878156-62ed-429b-adbe-2ea065f39a59 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.400632305Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.400796269Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"uri":"/crucible/0/regions","method":"POST","req_id":"66afa007-e23d-4f98-8dde-b8ebd49fe253","remote_addr":"[fd00:1122:3344:101::c]:57053","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":215,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.40081451Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_e4d000b9-898b-4a52-af6d-9050ee76dbb8/crucible/regions/22878156-62ed-429b-adbe-2ea065f39a59","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.407212894Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_e4d000b9-898b-4a52-af6d-9050ee76dbb8/crucible/regions/22878156-62ed-429b-adbe-2ea065f39a59","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.407229455Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"22878156-62ed-429b-adbe-2ea065f39a59\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/22878156-62ed-429b-adbe-2ea065f39a59\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.455929216Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"region":"22878156-62ed-429b-adbe-2ea065f39a59","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"22878156-62ed-429b-adbe-2ea065f39a59\" \"--data\" \"/data/regions/22878156-62ed-429b-adbe-2ea065f39a59\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.455964917Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"region":"22878156-62ed-429b-adbe-2ea065f39a59","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.527939175Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"region":"22878156-62ed-429b-adbe-2ea065f39a59","component":"worker"} {"msg":"region 22878156-62ed-429b-adbe-2ea065f39a59 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.527974526Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.528091589Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-22878156-62ed-429b-adbe-2ea065f39a59","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.528475149Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-22878156-62ed-429b-adbe-2ea065f39a59","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.531218349Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.537587673Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-22878156-62ed-429b-adbe-2ea065f39a59","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.540500297Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/22878156-62ed-429b-adbe-2ea065f39a59","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.540725643Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.540769904Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.540807955Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.540842936Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.544131301Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.553652705Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:03.655369038Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"uri":"/crucible/0/regions","method":"POST","req_id":"2262d5cb-559f-45fe-ab92-37fdc95147b2","remote_addr":"[fd00:1122:3344:101::c]:57053","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":62,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.62677311Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"uri":"/crucible/0/regions/22878156-62ed-429b-adbe-2ea065f39a59","method":"GET","req_id":"f335552a-02e1-414a-8184-f53f9b47717a","remote_addr":"[fd00:1122:3344:101::c]:57053","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":200} {"msg":"path is \"/data/regions/22878156-62ed-429b-adbe-2ea065f39a59\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.630955767Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"datafile"} {"msg":"dataset is oxp_e4d000b9-898b-4a52-af6d-9050ee76dbb8/crucible/regions/22878156-62ed-429b-adbe-2ea065f39a59","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.641611021Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"datafile"} {"msg":"region 22878156-62ed-429b-adbe-2ea065f39a59 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.648499828Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.648745104Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"uri":"/crucible/0/regions/22878156-62ed-429b-adbe-2ea065f39a59","method":"DELETE","req_id":"67e473e9-3d99-41dc-a532-740ae7c05ca6","remote_addr":"[fd00:1122:3344:101::c]:57053","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":17851,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.648763395Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-22878156-62ed-429b-adbe-2ea065f39a59 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.649402571Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.651745631Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"uri":"/crucible/0/regions/22878156-62ed-429b-adbe-2ea065f39a59","method":"GET","req_id":"af38425b-32ba-4984-90a3-daa655b86321","remote_addr":"[fd00:1122:3344:101::c]:57053","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":31,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.6579209Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_e4d000b9-898b-4a52-af6d-9050ee76dbb8/crucible/regions/22878156-62ed-429b-adbe-2ea065f39a59\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.665502205Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"region":"22878156-62ed-429b-adbe-2ea065f39a59","component":"worker"} {"msg":"region 22878156-62ed-429b-adbe-2ea065f39a59 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.71710977Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-24T04:55:29.800274315Z","hostname":"oxz_crucible_532b6f43-2474-4b72-90e8-74d7d680dbd8","pid":23822,"uri":"/crucible/0/regions/22878156-62ed-429b-adbe-2ea065f39a59","method":"GET","req_id":"916020ec-19ae-44f5-9de3-1cdbdf2316d8","remote_addr":"[fd00:1122:3344:101::c]:57053","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":42,"response_code":200}