[ Apr 14 23:30:37 Enabled. ] [ Apr 14 23:30:37 Rereading configuration. ] [ Apr 14 23:30:39 Rereading configuration. ] [ Apr 14 23:30:46 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.720661508Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616} {"msg":"dataset: \"oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.721450889Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616} {"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.72146637Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.72147349Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.734934647Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:30:46.817484416Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.279374593Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:39668"} {"msg":"region ade60330-59c1-4c05-b024-180adc9054e8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.306888093Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.309282598Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.309299488Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"uri":"/crucible/0/regions","method":"POST","req_id":"d05a2f91-f244-4b9d-95a2-a67715d9516c","remote_addr":"[fd00:1122:3344:101::c]:39668","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":15101,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.317348727Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.317364778Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"ade60330-59c1-4c05-b024-180adc9054e8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ade60330-59c1-4c05-b024-180adc9054e8\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.359099315Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"region":"ade60330-59c1-4c05-b024-180adc9054e8","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ade60330-59c1-4c05-b024-180adc9054e8\" \"--data\" \"/data/regions/ade60330-59c1-4c05-b024-180adc9054e8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.361350676Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"region":"ade60330-59c1-4c05-b024-180adc9054e8","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.507563359Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"uri":"/crucible/0/regions","method":"POST","req_id":"e52f6704-e960-4ed1-bc12-929168ef86ee","remote_addr":"[fd00:1122:3344:101::c]:39668","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":62,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.715818222Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"region":"ade60330-59c1-4c05-b024-180adc9054e8","component":"worker"} {"msg":"region ade60330-59c1-4c05-b024-180adc9054e8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.715866304Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.715937905Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-ade60330-59c1-4c05-b024-180adc9054e8","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.716285475Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ade60330-59c1-4c05-b024-180adc9054e8","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.731422607Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.746174219Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ade60330-59c1-4c05-b024-180adc9054e8","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.749610713Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ade60330-59c1-4c05-b024-180adc9054e8","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.75023399Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.750301992Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.750350273Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.750392754Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.754572138Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:37.758730201Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:31:38.178454575Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"uri":"/crucible/0/regions","method":"POST","req_id":"ef79b1a3-d8ea-4a86-b9b1-9138e4823400","remote_addr":"[fd00:1122:3344:101::c]:39668","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":56,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.045399471Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:57734"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.045739Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"uri":"/crucible/0/regions/ade60330-59c1-4c05-b024-180adc9054e8","method":"GET","req_id":"d210d2b4-0599-4f34-9498-6439c7108128","remote_addr":"[fd00:1122:3344:101::c]:57734","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":55,"response_code":200} {"msg":"path is \"/data/regions/ade60330-59c1-4c05-b024-180adc9054e8\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.049190024Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"dataset is oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.059809133Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"snapshot is oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8@18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.065889419Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8\", \"18ff68e4-f03f-48ec-8b7a-dfb1c4879220\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.06591827Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.075550462Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"uri":"/crucible/0/regions/ade60330-59c1-4c05-b024-180adc9054e8/snapshots/18ff68e4-f03f-48ec-8b7a-dfb1c4879220","method":"GET","req_id":"6317ae48-1351-4a05-a374-43cefb6af30d","remote_addr":"[fd00:1122:3344:101::c]:57734","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":26375,"response_code":200} {"msg":"path is \"/data/regions/ade60330-59c1-4c05-b024-180adc9054e8\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.079294734Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"dataset is oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.086004347Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"snapshot is oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8@18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.091722673Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8\", \"18ff68e4-f03f-48ec-8b7a-dfb1c4879220\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.091750073Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot ade60330-59c1-4c05-b024-180adc9054e8-18ff68e4-f03f-48ec-8b7a-dfb1c4879220 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.103286248Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"applying SMF actions for region ade60330-59c1-4c05-b024-180adc9054e8 running snapshot 18ff68e4-f03f-48ec-8b7a-dfb1c4879220 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.104289225Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.104301395Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"uri":"/crucible/0/regions/ade60330-59c1-4c05-b024-180adc9054e8/snapshots/18ff68e4-f03f-48ec-8b7a-dfb1c4879220/run","method":"POST","req_id":"69051191-b995-4268-bec9-4e8e9fb3a296","remote_addr":"[fd00:1122:3344:101::c]:57734","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25014,"response_code":200} {"msg":"creating missing snapshot instance snapshot-ade60330-59c1-4c05-b024-180adc9054e8-18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.125104462Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-ade60330-59c1-4c05-b024-180adc9054e8-18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.132599986Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-ade60330-59c1-4c05-b024-180adc9054e8-18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.168596767Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ade60330-59c1-4c05-b024-180adc9054e8/.zfs/snapshot/18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.168789732Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.168828383Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.168866524Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.168900365Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.168941856Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.18193591Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.198858101Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"worker"} {"msg":"region ade60330-59c1-4c05-b024-180adc9054e8 running snapshot 18ff68e4-f03f-48ec-8b7a-dfb1c4879220 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.198874712Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"path is \"/data/regions/ade60330-59c1-4c05-b024-180adc9054e8\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.339892113Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"dataset is oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.345884266Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"datafile"} {"msg":"snapshot is oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8@18ff68e4-f03f-48ec-8b7a-dfb1c4879220","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.351111139Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_2932c4b7-58b5-4416-9c2c-4248df98ee34/crucible/regions/ade60330-59c1-4c05-b024-180adc9054e8\", \"18ff68e4-f03f-48ec-8b7a-dfb1c4879220\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.351133749Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-14T23:42:16.357543054Z","hostname":"oxz_crucible_a0e48f23-4741-4644-bcf5-d112f843e9ea","pid":28616,"uri":"/crucible/0/regions/ade60330-59c1-4c05-b024-180adc9054e8/snapshots/18ff68e4-f03f-48ec-8b7a-dfb1c4879220/run","method":"POST","req_id":"6b9c8b8a-da1f-4f05-99d4-3538af318bf6","remote_addr":"[fd00:1122:3344:101::c]:57734","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17705,"response_code":200}