[ May 23 04:50:04 Enabled. ]
[ May 23 04:50:04 Rereading configuration. ]
[ May 23 04:50:07 Rereading configuration. ]
[ May 23 04:50:12 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_4a258f03-f13c-4995-91d3-5bc85733af75/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 3c1708d86e10f0370807388a1efe092edd99d431\nCommit timestamp: 2026-04-27T19:14:23.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:12.936258484Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684}
{"msg":"dataset: \"oxp_4a258f03-f13c-4995-91d3-5bc85733af75/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:12.936449109Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:12.936459499Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:12.93646511Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:12.941465213Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:13.042975307Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.447232139Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:45678"}
{"msg":"region 1e814c4e-97a2-40d9-9e30-e104ae91b92f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.486666427Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.487193371Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.487251933Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"uri":"/crucible/0/regions","method":"POST","req_id":"89a3c362-6ee3-4cc9-873f-8af16f60a504","remote_addr":"[fd00:1122:3344:101::c]:45678","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":21840,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_4a258f03-f13c-4995-91d3-5bc85733af75/crucible/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.496954493Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_4a258f03-f13c-4995-91d3-5bc85733af75/crucible/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.496974004Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1e814c4e-97a2-40d9-9e30-e104ae91b92f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.601994521Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"region":"1e814c4e-97a2-40d9-9e30-e104ae91b92f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1e814c4e-97a2-40d9-9e30-e104ae91b92f\" \"--data\" \"/data/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.602073673Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"region":"1e814c4e-97a2-40d9-9e30-e104ae91b92f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.834842567Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"uri":"/crucible/0/regions","method":"POST","req_id":"be2ba7b0-3293-41ee-94a0-d3fce2e32ab6","remote_addr":"[fd00:1122:3344:101::c]:45678","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.903711245Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"region":"1e814c4e-97a2-40d9-9e30-e104ae91b92f","component":"worker"}
{"msg":"region 1e814c4e-97a2-40d9-9e30-e104ae91b92f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.903774006Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.904005202Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1e814c4e-97a2-40d9-9e30-e104ae91b92f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.906327835Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1e814c4e-97a2-40d9-9e30-e104ae91b92f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.916019165Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.927573914Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1e814c4e-97a2-40d9-9e30-e104ae91b92f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.933126063Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.933836613Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.935874897Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.935927069Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.93596544Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.948749343Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:39.952982506Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:40.126950383Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"uri":"/crucible/0/regions","method":"POST","req_id":"3dbbfb57-a554-4912-976d-4df2fec8a232","remote_addr":"[fd00:1122:3344:101::c]:45678","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":88,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.16942238Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"uri":"/crucible/0/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f","method":"GET","req_id":"72c21ba5-ba48-4162-9138-d7e2259cd104","remote_addr":"[fd00:1122:3344:101::c]:45678","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":2106,"response_code":200}
{"msg":"path is \"/data/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.174339881Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"datafile"}
{"msg":"dataset is oxp_4a258f03-f13c-4995-91d3-5bc85733af75/crucible/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.183071286Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"datafile"}
{"msg":"region 1e814c4e-97a2-40d9-9e30-e104ae91b92f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.19105552Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.191216444Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"uri":"/crucible/0/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f","method":"DELETE","req_id":"88c7a226-cbc3-4945-a718-09ffa05bf302","remote_addr":"[fd00:1122:3344:101::c]:45678","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":16905,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.191276186Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-1e814c4e-97a2-40d9-9e30-e104ae91b92f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.192112388Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.195687764Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"uri":"/crucible/0/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f","method":"GET","req_id":"8b78119d-e1f8-4cd6-9e45-e5ac58622f3a","remote_addr":"[fd00:1122:3344:101::c]:45678","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":37,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.202034264Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_4a258f03-f13c-4995-91d3-5bc85733af75/crucible/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.210873432Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"region":"1e814c4e-97a2-40d9-9e30-e104ae91b92f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.276284266Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"uri":"/crucible/0/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f","method":"GET","req_id":"cb08418b-4337-41d5-b159-3a969989fe0e","remote_addr":"[fd00:1122:3344:101::c]:45678","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"region 1e814c4e-97a2-40d9-9e30-e104ae91b92f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.32001388Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-23T04:50:41.349434269Z","hostname":"oxz_crucible_dd06b0f6-54e8-45f9-8af4-e5176a6762c7","pid":25684,"uri":"/crucible/0/regions/1e814c4e-97a2-40d9-9e30-e104ae91b92f","method":"GET","req_id":"bffa28fa-48fb-49a6-b1d2-1c23055f5fec","remote_addr":"[fd00:1122:3344:101::c]:45678","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":38,"response_code":200}