[ Feb 24 15:43:51 Enabled. ]
[ Feb 24 15:43:51 Rereading configuration. ]
[ Feb 24 15:43:54 Rereading configuration. ]
[ Feb 24 15:43:58 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6c2c15d5-9bca-4ded-84d7-468f8ccadf49/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:43:59.161721573Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688}
{"msg":"dataset: \"oxp_6c2c15d5-9bca-4ded-84d7-468f8ccadf49/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:43:59.16251821Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:43:59.1625334Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:43:59.16254131Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:43:59.179362095Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:43:59.260329436Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:55.765406622Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:64684"}
{"msg":"region 4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:55.788543931Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:55.790995223Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:55.791023383Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"uri":"/crucible/0/regions","method":"POST","req_id":"2524e135-f8d5-484b-acbb-764927754c74","remote_addr":"[fd00:1122:3344:101::b]:64684","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":13030,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_6c2c15d5-9bca-4ded-84d7-468f8ccadf49/crucible/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:55.798407349Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_6c2c15d5-9bca-4ded-84d7-468f8ccadf49/crucible/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:55.798424029Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:55.846690549Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"region":"4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712\" \"--data\" \"/data/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:55.84863461Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"region":"4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.049315289Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"uri":"/crucible/0/regions","method":"POST","req_id":"f6a0466d-d45c-4b4a-bc45-a44bf762a740","remote_addr":"[fd00:1122:3344:101::b]:64684","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.198296176Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"region":"4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","component":"worker"}
{"msg":"region 4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.198345557Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.198430179Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.198774926Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.208768837Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.223023548Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.226353799Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.227120985Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.227189896Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.227232067Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.227271028Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.230920235Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.235029872Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:56.691006113Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"uri":"/crucible/0/regions","method":"POST","req_id":"4d68ad7a-9ec2-4863-8c8f-a850db3b57a8","remote_addr":"[fd00:1122:3344:101::b]:64684","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":71,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.268121292Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"uri":"/crucible/0/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","method":"GET","req_id":"5bfd5acb-80f2-4081-a71a-41e102574827","remote_addr":"[fd00:1122:3344:101::b]:64684","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":90,"response_code":200}
{"msg":"path is \"/data/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.271000703Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"datafile"}
{"msg":"dataset is oxp_6c2c15d5-9bca-4ded-84d7-468f8ccadf49/crucible/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.279492992Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"datafile"}
{"msg":"region 4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.287681535Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.287798638Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"uri":"/crucible/0/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","method":"DELETE","req_id":"fbc1f639-d41c-406f-808d-7ba35a463d8f","remote_addr":"[fd00:1122:3344:101::b]:64684","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":16828,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.287824548Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.288433461Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.29076413Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"uri":"/crucible/0/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","method":"GET","req_id":"b164a47d-8073-483d-96cc-a90b4d25548a","remote_addr":"[fd00:1122:3344:101::b]:64684","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":31,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.29310019Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6c2c15d5-9bca-4ded-84d7-468f8ccadf49/crucible/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712\"","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.302551689Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"region":"4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.350071803Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"uri":"/crucible/0/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","method":"GET","req_id":"da340549-ee25-4446-a0f7-f38899af5391","remote_addr":"[fd00:1122:3344:101::b]:64684","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"region 4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.363114049Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-02-24T15:44:57.453223202Z","hostname":"oxz_crucible_ecda6681-aa89-402c-bf81-187778d07fe2","pid":23688,"uri":"/crucible/0/regions/4f1f5bd8-ea1c-4c7a-9316-cd84fa1e1712","method":"GET","req_id":"a39a5b26-eb39-4333-ad0c-032d103b9f29","remote_addr":"[fd00:1122:3344:101::b]:64684","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":38,"response_code":200}