[ Mar 10 17:17:03 Enabled. ] [ Mar 10 17:17:03 Rereading configuration. ] [ Mar 10 17:17:06 Rereading configuration. ] [ Mar 10 17:17:11 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_40b4dd9b-fc90-4a52-a027-09694258b518/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:17:12.028689836Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265} {"msg":"dataset: \"oxp_40b4dd9b-fc90-4a52-a027-09694258b518/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:17:12.029438539Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265} {"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:17:12.029448899Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:17:12.029454279Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:17:12.04192543Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:17:12.165841868Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:45.753332342Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:48398"} {"msg":"region eda0cd31-3290-406f-a011-4c5de79a2f95 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:45.799826572Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:45.803518351Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:45.803535131Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"uri":"/crucible/0/regions","method":"POST","req_id":"04d32279-359b-40e8-a08b-c729a70d5fff","remote_addr":"[fd00:1122:3344:101::c]:48398","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20623,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_40b4dd9b-fc90-4a52-a027-09694258b518/crucible/regions/eda0cd31-3290-406f-a011-4c5de79a2f95","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:45.814199413Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_40b4dd9b-fc90-4a52-a027-09694258b518/crucible/regions/eda0cd31-3290-406f-a011-4c5de79a2f95","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:45.814216344Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"eda0cd31-3290-406f-a011-4c5de79a2f95\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/eda0cd31-3290-406f-a011-4c5de79a2f95\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:45.856741159Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"region":"eda0cd31-3290-406f-a011-4c5de79a2f95","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"eda0cd31-3290-406f-a011-4c5de79a2f95\" \"--data\" \"/data/regions/eda0cd31-3290-406f-a011-4c5de79a2f95\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:45.858983395Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"region":"eda0cd31-3290-406f-a011-4c5de79a2f95","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.113093963Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"uri":"/crucible/0/regions","method":"POST","req_id":"87edde4d-b67f-458e-81ba-3ae218fd4ce1","remote_addr":"[fd00:1122:3344:101::c]:48398","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":58,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.216913307Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"region":"eda0cd31-3290-406f-a011-4c5de79a2f95","component":"worker"} {"msg":"region eda0cd31-3290-406f-a011-4c5de79a2f95 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.216968628Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.21704574Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-eda0cd31-3290-406f-a011-4c5de79a2f95","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.217373335Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-eda0cd31-3290-406f-a011-4c5de79a2f95","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.233633817Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.259826349Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-eda0cd31-3290-406f-a011-4c5de79a2f95","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.262504713Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eda0cd31-3290-406f-a011-4c5de79a2f95","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.263330226Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.263401817Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.263448548Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.263489798Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.267086607Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.27104424Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:46.446261735Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"uri":"/crucible/0/regions","method":"POST","req_id":"f2124935-e5b0-4585-b3be-a8255ebe4b03","remote_addr":"[fd00:1122:3344:101::c]:48398","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"region ec11ba8b-ee77-406d-a712-6c7f7de885fd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.468340509Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.468497572Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"uri":"/crucible/0/regions","method":"POST","req_id":"447bab86-8b31-44f9-9a88-a9c8b37e6c39","remote_addr":"[fd00:1122:3344:101::c]:48398","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":209,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.468524102Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_40b4dd9b-fc90-4a52-a027-09694258b518/crucible/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.475081348Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_40b4dd9b-fc90-4a52-a027-09694258b518/crucible/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.475094568Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"ec11ba8b-ee77-406d-a712-6c7f7de885fd\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.517477942Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"region":"ec11ba8b-ee77-406d-a712-6c7f7de885fd","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ec11ba8b-ee77-406d-a712-6c7f7de885fd\" \"--data\" \"/data/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.517514102Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"region":"ec11ba8b-ee77-406d-a712-6c7f7de885fd","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.58867777Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"region":"ec11ba8b-ee77-406d-a712-6c7f7de885fd","component":"worker"} {"msg":"region ec11ba8b-ee77-406d-a712-6c7f7de885fd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.58871467Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.588797872Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-ec11ba8b-ee77-406d-a712-6c7f7de885fd","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.589122757Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ec11ba8b-ee77-406d-a712-6c7f7de885fd","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.592208527Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.599350702Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ec11ba8b-ee77-406d-a712-6c7f7de885fd","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.601784851Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.602076386Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.602120287Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.602162378Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.602197748Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.605793386Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.61660873Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:28:52.631848336Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"uri":"/crucible/0/regions","method":"POST","req_id":"f0ca11cf-f70f-4e3b-800e-e5aa2746cfb3","remote_addr":"[fd00:1122:3344:101::c]:48398","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":61,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.487459373Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"uri":"/crucible/0/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd","method":"GET","req_id":"a8a3bdac-15d4-4fe7-b965-f68b4cb76bad","remote_addr":"[fd00:1122:3344:101::c]:48398","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":76,"response_code":200} {"msg":"path is \"/data/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.490750576Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"datafile"} {"msg":"dataset is oxp_40b4dd9b-fc90-4a52-a027-09694258b518/crucible/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.500402212Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"datafile"} {"msg":"region ec11ba8b-ee77-406d-a712-6c7f7de885fd state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.508241508Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.508445382Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"uri":"/crucible/0/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd","method":"DELETE","req_id":"a099d2d6-bf16-4189-a8f4-60c3848e6a1e","remote_addr":"[fd00:1122:3344:101::c]:48398","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17725,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.508490792Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-ec11ba8b-ee77-406d-a712-6c7f7de885fd (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.50961629Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.511723364Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"uri":"/crucible/0/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd","method":"GET","req_id":"550d4a4b-ee14-4d36-9b7b-1523d87fc08b","remote_addr":"[fd00:1122:3344:101::c]:48398","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":32,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.515997433Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_40b4dd9b-fc90-4a52-a027-09694258b518/crucible/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.522452237Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"region":"ec11ba8b-ee77-406d-a712-6c7f7de885fd","component":"worker"} {"msg":"region ec11ba8b-ee77-406d-a712-6c7f7de885fd state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.589764343Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-10T17:29:19.640082524Z","hostname":"oxz_crucible_124569ef-800a-4715-941d-b20cc0507ebe","pid":28265,"uri":"/crucible/0/regions/ec11ba8b-ee77-406d-a712-6c7f7de885fd","method":"GET","req_id":"92566601-d5f4-42bc-8d2d-a0a7fad9941b","remote_addr":"[fd00:1122:3344:101::c]:48398","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":51,"response_code":200}