{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:50.827852275Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:51122"} {"msg":"region e1a9488c-236c-45e5-b1ef-0fe3fb097c5b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:50.874788676Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:50.877743533Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:50.877757084Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"uri":"/crucible/0/regions","method":"POST","req_id":"41af68b9-79ad-4063-8958-592fac87cf4b","remote_addr":"[fd00:1122:3344:101::b]:51122","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19314,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_0746f33e-d948-49f2-acfd-44d8cac173bf/crucible/regions/e1a9488c-236c-45e5-b1ef-0fe3fb097c5b","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:50.890793766Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_0746f33e-d948-49f2-acfd-44d8cac173bf/crucible/regions/e1a9488c-236c-45e5-b1ef-0fe3fb097c5b","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:50.890808986Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e1a9488c-236c-45e5-b1ef-0fe3fb097c5b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e1a9488c-236c-45e5-b1ef-0fe3fb097c5b\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:50.961662965Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"region":"e1a9488c-236c-45e5-b1ef-0fe3fb097c5b","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e1a9488c-236c-45e5-b1ef-0fe3fb097c5b\" \"--data\" \"/data/regions/e1a9488c-236c-45e5-b1ef-0fe3fb097c5b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:50.964041317Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"region":"e1a9488c-236c-45e5-b1ef-0fe3fb097c5b","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.167595257Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"uri":"/crucible/0/regions","method":"POST","req_id":"f0793fed-253a-4f78-bf3f-0e1cc0940506","remote_addr":"[fd00:1122:3344:101::b]:51122","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":62,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.343761029Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"region":"e1a9488c-236c-45e5-b1ef-0fe3fb097c5b","component":"worker"} {"msg":"region e1a9488c-236c-45e5-b1ef-0fe3fb097c5b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.34380702Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.343879522Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e1a9488c-236c-45e5-b1ef-0fe3fb097c5b","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.344212131Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e1a9488c-236c-45e5-b1ef-0fe3fb097c5b","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.357044607Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.380062361Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e1a9488c-236c-45e5-b1ef-0fe3fb097c5b","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.382889816Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e1a9488c-236c-45e5-b1ef-0fe3fb097c5b","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.383514212Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.383580183Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.383621155Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.383656225Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.387230869Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.391591224Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:51.919361049Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"uri":"/crucible/0/regions","method":"POST","req_id":"c93809e4-7cc6-4b38-864b-cdb634986833","remote_addr":"[fd00:1122:3344:101::b]:51122","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":71,"response_code":200} {"msg":"region f0677687-8bb7-4c25-9b32-a6cb387402ac state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.319613303Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.319795838Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"uri":"/crucible/0/regions","method":"POST","req_id":"a29bd237-a559-4688-9c4e-2b68368e8a7f","remote_addr":"[fd00:1122:3344:101::b]:51122","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":235,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.319815609Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_0746f33e-d948-49f2-acfd-44d8cac173bf/crucible/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.326232297Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_0746f33e-d948-49f2-acfd-44d8cac173bf/crucible/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.326247247Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f0677687-8bb7-4c25-9b32-a6cb387402ac\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.390514573Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"region":"f0677687-8bb7-4c25-9b32-a6cb387402ac","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f0677687-8bb7-4c25-9b32-a6cb387402ac\" \"--data\" \"/data/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.390560154Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"region":"f0677687-8bb7-4c25-9b32-a6cb387402ac","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.471328033Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"region":"f0677687-8bb7-4c25-9b32-a6cb387402ac","component":"worker"} {"msg":"region f0677687-8bb7-4c25-9b32-a6cb387402ac state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.471363014Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.471482307Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f0677687-8bb7-4c25-9b32-a6cb387402ac","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.475215915Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f0677687-8bb7-4c25-9b32-a6cb387402ac","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.47806531Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.485711221Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f0677687-8bb7-4c25-9b32-a6cb387402ac","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.489153401Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.489441659Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.48949375Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.489537031Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.489578802Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.493782162Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.498167968Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:06:58.505551041Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"uri":"/crucible/0/regions","method":"POST","req_id":"8ea8fb6a-fb33-4faa-be88-739ae8ca129e","remote_addr":"[fd00:1122:3344:101::b]:51122","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":59,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.729871953Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"uri":"/crucible/0/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac","method":"GET","req_id":"3c5d181e-7097-4546-b02d-17345ce90f94","remote_addr":"[fd00:1122:3344:101::b]:51122","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":77,"response_code":200} {"msg":"path is \"/data/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.734491524Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"datafile"} {"msg":"dataset is oxp_0746f33e-d948-49f2-acfd-44d8cac173bf/crucible/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.745094322Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"datafile"} {"msg":"region f0677687-8bb7-4c25-9b32-a6cb387402ac state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.751620023Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.75223696Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.75224692Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"uri":"/crucible/0/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac","method":"DELETE","req_id":"05a53b29-f3ef-47bd-a44f-bbe1f4862251","remote_addr":"[fd00:1122:3344:101::b]:51122","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":17806,"response_code":204} {"msg":"disabling downstairs instance: downstairs-f0677687-8bb7-4c25-9b32-a6cb387402ac (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.752747473Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.755358482Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"uri":"/crucible/0/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac","method":"GET","req_id":"b092a6e6-8b4f-4d39-9f2d-3199ecbb242c","remote_addr":"[fd00:1122:3344:101::b]:51122","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":35,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.777533983Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_0746f33e-d948-49f2-acfd-44d8cac173bf/crucible/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.783953842Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"region":"f0677687-8bb7-4c25-9b32-a6cb387402ac","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.874155998Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"uri":"/crucible/0/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac","method":"GET","req_id":"7b5276d5-6523-4fbb-b424-670f08bdb3c6","remote_addr":"[fd00:1122:3344:101::b]:51122","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":62,"response_code":200} {"msg":"region f0677687-8bb7-4c25-9b32-a6cb387402ac state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:26.90204397Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-05T00:07:27.021438662Z","hostname":"oxz_crucible_2bec9265-c13a-4604-8486-dfd8eb2adf00","pid":23179,"uri":"/crucible/0/regions/f0677687-8bb7-4c25-9b32-a6cb387402ac","method":"GET","req_id":"e5f24a78-11b2-4248-a6b5-308025f59605","remote_addr":"[fd00:1122:3344:101::b]:51122","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":42,"response_code":200}