[ Jan 23 16:58:33 Enabled. ] [ Jan 23 16:58:33 Rereading configuration. ] [ Jan 23 16:58:34 Rereading configuration. ] [ Jan 23 16:58:37 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_dfe05d41-2afe-429f-8b84-9813985a5691/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:58:37.353699838Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047} {"msg":"dataset: \"oxp_dfe05d41-2afe-429f-8b84-9813985a5691/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:58:37.353934911Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047} {"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:58:37.353948571Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:58:37.353956451Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:58:37.367999731Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:58:37.475950406Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.368639141Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53743"} {"msg":"region db5dd0ce-4d23-4bdc-a12e-0b13884ed615 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.395455428Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.398255058Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.398316898Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"uri":"/crucible/0/regions","method":"POST","req_id":"9f43e823-4f0a-41e9-b011-c5120e63c795","remote_addr":"[fd00:1122:3344:101::a]:53743","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":14873,"response_code":200} {"msg":"zfs set reservation of 1342177280 for oxp_dfe05d41-2afe-429f-8b84-9813985a5691/crucible/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.406834709Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_dfe05d41-2afe-429f-8b84-9813985a5691/crucible/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.406851539Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"db5dd0ce-4d23-4bdc-a12e-0b13884ed615\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.472039887Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"region":"db5dd0ce-4d23-4bdc-a12e-0b13884ed615","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"db5dd0ce-4d23-4bdc-a12e-0b13884ed615\" \"--data\" \"/data/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.475211951Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"region":"db5dd0ce-4d23-4bdc-a12e-0b13884ed615","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.568024463Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"uri":"/crucible/0/regions","method":"POST","req_id":"a723d42b-1bb0-4f1f-8567-f874c848b5d5","remote_addr":"[fd00:1122:3344:101::a]:53743","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":64,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.819825127Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"region":"db5dd0ce-4d23-4bdc-a12e-0b13884ed615","component":"worker"} {"msg":"region db5dd0ce-4d23-4bdc-a12e-0b13884ed615 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.819924758Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.820069109Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-db5dd0ce-4d23-4bdc-a12e-0b13884ed615","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.820396523Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-db5dd0ce-4d23-4bdc-a12e-0b13884ed615","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.825026852Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.83697523Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-db5dd0ce-4d23-4bdc-a12e-0b13884ed615","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.841237236Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.841916623Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.841980954Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.842023424Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.842063845Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.846032077Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:13.850763268Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.204084486Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"uri":"/crucible/0/regions","method":"POST","req_id":"4882758e-e878-4711-9713-1117e3983ff3","remote_addr":"[fd00:1122:3344:101::a]:53743","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":60,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.6148352Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"uri":"/crucible/0/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615","method":"GET","req_id":"5537e5f5-b3b6-4e90-92f7-d134f9dce7b3","remote_addr":"[fd00:1122:3344:101::a]:53743","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":77,"response_code":200} {"msg":"path is \"/data/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.618345457Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"datafile"} {"msg":"dataset is oxp_dfe05d41-2afe-429f-8b84-9813985a5691/crucible/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.626814738Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"datafile"} {"msg":"region db5dd0ce-4d23-4bdc-a12e-0b13884ed615 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.635921205Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.636129908Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.636176118Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"uri":"/crucible/0/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615","method":"DELETE","req_id":"55c0bc9d-d6f0-4080-91cf-d27117305c5a","remote_addr":"[fd00:1122:3344:101::a]:53743","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":17812,"response_code":204} {"msg":"disabling downstairs instance: downstairs-db5dd0ce-4d23-4bdc-a12e-0b13884ed615 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.637041917Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.639246241Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"uri":"/crucible/0/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615","method":"GET","req_id":"782590ef-05c2-44aa-9e9a-1051d3b92428","remote_addr":"[fd00:1122:3344:101::a]:53743","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":30,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.641566756Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_dfe05d41-2afe-429f-8b84-9813985a5691/crucible/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.649792564Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"region":"db5dd0ce-4d23-4bdc-a12e-0b13884ed615","component":"worker"} {"msg":"region db5dd0ce-4d23-4bdc-a12e-0b13884ed615 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.724102108Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:14.739568424Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"uri":"/crucible/0/regions/db5dd0ce-4d23-4bdc-a12e-0b13884ed615","method":"GET","req_id":"4c04225b-1671-48d4-bb5e-2cae6c672a43","remote_addr":"[fd00:1122:3344:101::a]:53743","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":48,"response_code":200} {"msg":"region b91f7043-1521-4c11-b97b-f748d654dfe3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.72129107Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.721496363Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.721562753Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"uri":"/crucible/0/regions","method":"POST","req_id":"4095f180-45e7-4169-845f-5e0cd9908426","remote_addr":"[fd00:1122:3344:101::a]:53743","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":256,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_dfe05d41-2afe-429f-8b84-9813985a5691/crucible/regions/b91f7043-1521-4c11-b97b-f748d654dfe3","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.727515257Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_dfe05d41-2afe-429f-8b84-9813985a5691/crucible/regions/b91f7043-1521-4c11-b97b-f748d654dfe3","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.727537457Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"b91f7043-1521-4c11-b97b-f748d654dfe3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b91f7043-1521-4c11-b97b-f748d654dfe3\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.791840355Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"region":"b91f7043-1521-4c11-b97b-f748d654dfe3","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b91f7043-1521-4c11-b97b-f748d654dfe3\" \"--data\" \"/data/regions/b91f7043-1521-4c11-b97b-f748d654dfe3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.791907166Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"region":"b91f7043-1521-4c11-b97b-f748d654dfe3","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.862902295Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"region":"b91f7043-1521-4c11-b97b-f748d654dfe3","component":"worker"} {"msg":"region b91f7043-1521-4c11-b97b-f748d654dfe3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.862930836Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.863154048Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-b91f7043-1521-4c11-b97b-f748d654dfe3","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.864386851Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b91f7043-1521-4c11-b97b-f748d654dfe3","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.867963269Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.875282968Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b91f7043-1521-4c11-b97b-f748d654dfe3","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.878635004Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b91f7043-1521-4c11-b97b-f748d654dfe3","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.878924647Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.878968717Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.879009268Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.879043018Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.882935919Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.887562679Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-01-23T16:59:16.950484922Z","hostname":"oxz_crucible_2871e122-1a36-4906-b731-1060f87737cc","pid":22047,"uri":"/crucible/0/regions","method":"POST","req_id":"3e0581c7-ed20-4000-838e-8b91e7c88508","remote_addr":"[fd00:1122:3344:101::a]:53743","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":58,"response_code":200}