[ Mar 18 01:36:53 Enabled. ]
[ Mar 18 01:36:53 Rereading configuration. ]
[ Mar 18 01:36:55 Rereading configuration. ]
[ Mar 18 01:37:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ef33e8d9-90bc-4420-bc4b-a1681ba5cff1/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:37:01.257916885Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486}
{"msg":"dataset: \"oxp_ef33e8d9-90bc-4420-bc4b-a1681ba5cff1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:37:01.258142068Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:37:01.258155879Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:37:01.258162999Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:37:01.271117838Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:37:01.374212317Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.119292325Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:64440"}
{"msg":"region c15a99f1-cee3-412d-88b1-dd777d557d2a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.1484814Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.151259996Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.151280936Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"uri":"/crucible/0/regions","method":"POST","req_id":"5631a88c-ff7d-4d59-bce5-77e54188ba72","remote_addr":"[fd00:1122:3344:101::c]:64440","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":15677,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_ef33e8d9-90bc-4420-bc4b-a1681ba5cff1/crucible/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.160664863Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ef33e8d9-90bc-4420-bc4b-a1681ba5cff1/crucible/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.160678743Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c15a99f1-cee3-412d-88b1-dd777d557d2a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.219525901Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"region":"c15a99f1-cee3-412d-88b1-dd777d557d2a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c15a99f1-cee3-412d-88b1-dd777d557d2a\" \"--data\" \"/data/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.229165901Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"region":"c15a99f1-cee3-412d-88b1-dd777d557d2a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.428790079Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"uri":"/crucible/0/regions","method":"POST","req_id":"929be66f-88b7-4389-b7f1-ceb5b1623940","remote_addr":"[fd00:1122:3344:101::c]:64440","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.566552578Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"region":"c15a99f1-cee3-412d-88b1-dd777d557d2a","component":"worker"}
{"msg":"region c15a99f1-cee3-412d-88b1-dd777d557d2a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.566595539Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.566674Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c15a99f1-cee3-412d-88b1-dd777d557d2a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.567006065Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c15a99f1-cee3-412d-88b1-dd777d557d2a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.57691547Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.594700986Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c15a99f1-cee3-412d-88b1-dd777d557d2a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.597599364Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.598149223Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.598208404Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.598246475Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.598280705Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.603447731Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.609569653Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:16.797108899Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"uri":"/crucible/0/regions","method":"POST","req_id":"0c4e9c0e-0fea-4ebc-8127-00ac5d17d11d","remote_addr":"[fd00:1122:3344:101::c]:64440","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.26528658Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"uri":"/crucible/0/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a","method":"GET","req_id":"676a77a7-022a-423c-b023-4165914712fa","remote_addr":"[fd00:1122:3344:101::c]:64440","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":82,"response_code":200}
{"msg":"path is \"/data/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.268810979Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"datafile"}
{"msg":"dataset is oxp_ef33e8d9-90bc-4420-bc4b-a1681ba5cff1/crucible/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.277129417Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"datafile"}
{"msg":"region c15a99f1-cee3-412d-88b1-dd777d557d2a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.284392377Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.284508057Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"uri":"/crucible/0/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a","method":"DELETE","req_id":"2f56394d-4f47-40e4-ad27-33c862742389","remote_addr":"[fd00:1122:3344:101::c]:64440","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":15780,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.284530276Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-c15a99f1-cee3-412d-88b1-dd777d557d2a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.285502486Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.28935732Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"uri":"/crucible/0/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a","method":"GET","req_id":"a28cd94a-0527-4143-837f-c1dd6a9aaf49","remote_addr":"[fd00:1122:3344:101::c]:64440","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":31,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.289564563Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ef33e8d9-90bc-4420-bc4b-a1681ba5cff1/crucible/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.29780517Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"region":"c15a99f1-cee3-412d-88b1-dd777d557d2a","component":"worker"}
{"msg":"region c15a99f1-cee3-412d-88b1-dd777d557d2a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.360696006Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-18T01:38:17.430342763Z","hostname":"oxz_crucible_f2bb66fe-a7a9-4010-90a7-f385d305ef69","pid":23486,"uri":"/crucible/0/regions/c15a99f1-cee3-412d-88b1-dd777d557d2a","method":"GET","req_id":"ffe4527e-7316-4b2c-9fea-ec8e25034c02","remote_addr":"[fd00:1122:3344:101::c]:64440","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":51,"response_code":200}