[ Mar 2 04:20:07 Enabled. ] [ Mar 2 04:20:07 Rereading configuration. ] [ Mar 2 04:20:09 Rereading configuration. ] [ Mar 2 04:20:23 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:23.891001676Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709} {"msg":"dataset: \"oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:23.891839323Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709} {"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:23.891853663Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:23.891859573Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:23.90692875Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:24.042782108Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:55.605337398Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58748"} {"msg":"region 9506fe49-0d45-4525-8127-d06b0d5ebdd6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:55.632569758Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:55.635294051Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:55.635338521Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"uri":"/crucible/0/regions","method":"POST","req_id":"bcb6a770-aa3c-45b3-9e50-e272a363a496","remote_addr":"[fd00:1122:3344:101::a]:58748","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":16138,"response_code":200} {"msg":"zfs set reservation of 1342177280 for oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:55.643902514Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:55.643918894Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"9506fe49-0d45-4525-8127-d06b0d5ebdd6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:55.730839438Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"region":"9506fe49-0d45-4525-8127-d06b0d5ebdd6","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9506fe49-0d45-4525-8127-d06b0d5ebdd6\" \"--data\" \"/data/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:55.733271278Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"region":"9506fe49-0d45-4525-8127-d06b0d5ebdd6","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:55.827752296Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"uri":"/crucible/0/regions","method":"POST","req_id":"5a0deccd-f19f-4bc3-b35d-2a6b7e059e2a","remote_addr":"[fd00:1122:3344:101::a]:58748","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":90,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.088529889Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"region":"9506fe49-0d45-4525-8127-d06b0d5ebdd6","component":"worker"} {"msg":"region 9506fe49-0d45-4525-8127-d06b0d5ebdd6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.088615009Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.088872732Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-9506fe49-0d45-4525-8127-d06b0d5ebdd6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.089273535Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9506fe49-0d45-4525-8127-d06b0d5ebdd6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.094511539Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.111085819Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9506fe49-0d45-4525-8127-d06b0d5ebdd6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.114034704Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.114594969Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.114658989Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.11469938Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.11473311Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.118694823Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.123292172Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.579996269Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"uri":"/crucible/0/regions","method":"POST","req_id":"cb7e74a1-815a-4655-9af6-f61ad487ed5b","remote_addr":"[fd00:1122:3344:101::a]:58748","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":62,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.992051669Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"uri":"/crucible/0/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6","method":"GET","req_id":"7cb5059d-ef1f-4b54-a9cf-34d23b843130","remote_addr":"[fd00:1122:3344:101::a]:58748","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":94,"response_code":200} {"msg":"path is \"/data/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:56.995041714Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"datafile"} {"msg":"dataset is oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:57.003913229Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"datafile"} {"msg":"region 9506fe49-0d45-4525-8127-d06b0d5ebdd6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:57.011699715Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:57.011867486Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"uri":"/crucible/0/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6","method":"DELETE","req_id":"08244097-1644-4a9e-996d-786cd1b1e72f","remote_addr":"[fd00:1122:3344:101::a]:58748","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":16850,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:57.011912216Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-9506fe49-0d45-4525-8127-d06b0d5ebdd6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:57.012866084Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:57.014965972Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"uri":"/crucible/0/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6","method":"GET","req_id":"693d963c-0205-4492-ab08-7bc23cb26f41","remote_addr":"[fd00:1122:3344:101::a]:58748","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":32,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:57.023919818Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:57.033479278Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"region":"9506fe49-0d45-4525-8127-d06b0d5ebdd6","component":"worker"} {"msg":"region 9506fe49-0d45-4525-8127-d06b0d5ebdd6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:57.124997262Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:57.162811711Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"uri":"/crucible/0/regions/9506fe49-0d45-4525-8127-d06b0d5ebdd6","method":"GET","req_id":"0169e3d3-9f95-4362-b861-d270268ecdbb","remote_addr":"[fd00:1122:3344:101::a]:58748","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":55,"response_code":200} {"msg":"region 528b89cc-82c2-453d-8a03-39a2394fe2ca state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.482904743Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.483100275Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"uri":"/crucible/0/regions","method":"POST","req_id":"bc34ac8e-2d27-4947-882b-289e5906d01f","remote_addr":"[fd00:1122:3344:101::a]:58748","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":252,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.483164346Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.489696651Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_7cebad1b-7367-4689-8ae2-6b246f116772/crucible/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.489720051Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"528b89cc-82c2-453d-8a03-39a2394fe2ca\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.562656327Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"region":"528b89cc-82c2-453d-8a03-39a2394fe2ca","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"528b89cc-82c2-453d-8a03-39a2394fe2ca\" \"--data\" \"/data/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.562738888Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"region":"528b89cc-82c2-453d-8a03-39a2394fe2ca","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.636371859Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"region":"528b89cc-82c2-453d-8a03-39a2394fe2ca","component":"worker"} {"msg":"region 528b89cc-82c2-453d-8a03-39a2394fe2ca state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.63640069Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.636626922Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-528b89cc-82c2-453d-8a03-39a2394fe2ca","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.638001713Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-528b89cc-82c2-453d-8a03-39a2394fe2ca","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.641009889Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.648655063Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-528b89cc-82c2-453d-8a03-39a2394fe2ca","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.651748749Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/528b89cc-82c2-453d-8a03-39a2394fe2ca","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.652063022Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.652128033Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.652143383Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.652178133Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.656907243Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.662064276Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-02T04:20:59.849152617Z","hostname":"oxz_crucible_587c7efb-91ac-476b-a203-f512d838a728","pid":24709,"uri":"/crucible/0/regions","method":"POST","req_id":"ea8e2db1-1866-420c-871b-cdecede4bb42","remote_addr":"[fd00:1122:3344:101::a]:58748","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":59,"response_code":200}