[ Apr 22 02:09:57 Enabled. ]
[ Apr 22 02:09:57 Rereading configuration. ]
[ Apr 22 02:09:59 Rereading configuration. ]
[ Apr 22 02:10:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_494bca7a-0d10-4fa7-928a-7da87c331c4a/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:05.429819163Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757}
{"msg":"dataset: \"oxp_494bca7a-0d10-4fa7-928a-7da87c331c4a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:05.430667348Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:05.430678279Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:05.430683979Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:05.445027055Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:05.529973606Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.316429103Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54090"}
{"msg":"region 29305431-f1b0-4b78-bc6f-639ed5267f82 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.345205008Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.348397096Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.348470337Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"uri":"/crucible/0/regions","method":"POST","req_id":"1adf33f9-3ecb-42a6-ab52-11b1105138dd","remote_addr":"[fd00:1122:3344:101::c]:54090","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":15905,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_494bca7a-0d10-4fa7-928a-7da87c331c4a/crucible/regions/29305431-f1b0-4b78-bc6f-639ed5267f82","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.357158462Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_494bca7a-0d10-4fa7-928a-7da87c331c4a/crucible/regions/29305431-f1b0-4b78-bc6f-639ed5267f82","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.357175763Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"29305431-f1b0-4b78-bc6f-639ed5267f82\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/29305431-f1b0-4b78-bc6f-639ed5267f82\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.416389863Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"region":"29305431-f1b0-4b78-bc6f-639ed5267f82","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"29305431-f1b0-4b78-bc6f-639ed5267f82\" \"--data\" \"/data/regions/29305431-f1b0-4b78-bc6f-639ed5267f82\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.418969569Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"region":"29305431-f1b0-4b78-bc6f-639ed5267f82","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.522619475Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"uri":"/crucible/0/regions","method":"POST","req_id":"506637a8-ec18-42e4-882f-b08bac988911","remote_addr":"[fd00:1122:3344:101::c]:54090","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":66,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.764349553Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"region":"29305431-f1b0-4b78-bc6f-639ed5267f82","component":"worker"}
{"msg":"region 29305431-f1b0-4b78-bc6f-639ed5267f82 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.764416754Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.76474018Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-29305431-f1b0-4b78-bc6f-639ed5267f82","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.765062436Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-29305431-f1b0-4b78-bc6f-639ed5267f82","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.771348078Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.78483723Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-29305431-f1b0-4b78-bc6f-639ed5267f82","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.788149359Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/29305431-f1b0-4b78-bc6f-639ed5267f82","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.788732289Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.78879011Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.788828511Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.788864312Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.793744249Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.801216753Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:48.860900072Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"uri":"/crucible/0/regions","method":"POST","req_id":"9994dc6c-cd1c-4af4-ac1b-90fe737b26bd","remote_addr":"[fd00:1122:3344:101::c]:54090","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.872466291Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"uri":"/crucible/0/regions/29305431-f1b0-4b78-bc6f-639ed5267f82","method":"GET","req_id":"0b19e8ef-5ecf-4481-9fa4-e125f18a3ecf","remote_addr":"[fd00:1122:3344:101::c]:54090","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":83,"response_code":200}
{"msg":"path is \"/data/regions/29305431-f1b0-4b78-bc6f-639ed5267f82\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.877930419Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"datafile"}
{"msg":"dataset is oxp_494bca7a-0d10-4fa7-928a-7da87c331c4a/crucible/regions/29305431-f1b0-4b78-bc6f-639ed5267f82","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.886260518Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"datafile"}
{"msg":"region 29305431-f1b0-4b78-bc6f-639ed5267f82 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.894047288Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.894263752Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"uri":"/crucible/0/regions/29305431-f1b0-4b78-bc6f-639ed5267f82","method":"DELETE","req_id":"da98a9dd-a1c9-44e7-b77f-dbb992f8a997","remote_addr":"[fd00:1122:3344:101::c]:54090","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":16360,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.894316832Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-29305431-f1b0-4b78-bc6f-639ed5267f82 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.895211628Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.899962543Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.90088956Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"uri":"/crucible/0/regions/29305431-f1b0-4b78-bc6f-639ed5267f82","method":"GET","req_id":"bc437799-7823-432c-b932-01b7024a6382","remote_addr":"[fd00:1122:3344:101::c]:54090","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27,"response_code":200}
{"msg":"deleting zfs dataset \"oxp_494bca7a-0d10-4fa7-928a-7da87c331c4a/crucible/regions/29305431-f1b0-4b78-bc6f-639ed5267f82\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.908034248Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"region":"29305431-f1b0-4b78-bc6f-639ed5267f82","component":"worker"}
{"msg":"region 29305431-f1b0-4b78-bc6f-639ed5267f82 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:49.977069034Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-22T02:10:50.045788734Z","hostname":"oxz_crucible_71d879fa-9378-4383-9ebc-65eef9a2b190","pid":23757,"uri":"/crucible/0/regions/29305431-f1b0-4b78-bc6f-639ed5267f82","method":"GET","req_id":"e2d71000-057b-4d10-ae84-772cdeaa8f1f","remote_addr":"[fd00:1122:3344:101::c]:54090","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":55,"response_code":200}