[ Oct  3 21:06:49 Enabled. ]
[ Oct  3 21:06:49 Rereading configuration. ]
[ Oct  3 21:06:50 Rereading configuration. ]
[ Oct  3 21:06:59 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0a6d51c0-d0df-4812-9c5a-213b208d92aa/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 65ca41e821ef53ec9c28909357f23e3348169e4f\nCommit timestamp: 2025-09-17T20:55:30.000000000Z  branch: main\nrustc: 1.86.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:06:59.695613349Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195}
{"msg":"dataset: \"oxp_0a6d51c0-d0df-4812-9c5a-213b208d92aa/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:06:59.696372047Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:06:59.696396797Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:06:59.696403897Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:06:59.711332636Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:06:59.851907433Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:23.99723394Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61958"}
{"msg":"region 6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.040566667Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.047362506Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"uri":"/crucible/0/regions","method":"POST","req_id":"9dca57f7-772e-4e0b-ae9f-5e148580c0ec","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25631,"response_code":200}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.050148487Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_0a6d51c0-d0df-4812-9c5a-213b208d92aa/crucible/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.060045519Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_0a6d51c0-d0df-4812-9c5a-213b208d92aa/crucible/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.060074079Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.222979518Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"region":"6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a\" \"--data\" \"/data/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.225143692Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"region":"6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.36753228Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"uri":"/crucible/0/regions","method":"POST","req_id":"49efce19-d4e6-4b02-9f28-96814b75b693","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":65,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.376278718Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"region":"6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","component":"worker"}
{"msg":"region 6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.376340719Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.376586242Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.376995647Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.38263768Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.405498679Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.415223298Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.415971607Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.416028817Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.416075298Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.416113378Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.422156547Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:24.428096994Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.029586763Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"uri":"/crucible/0/regions","method":"POST","req_id":"7a5cd3de-6fd7-4efe-9669-9a8d27a70cf6","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":47,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.685217623Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"uri":"/crucible/0/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","method":"GET","req_id":"57cb1c60-4caf-4c40-a040-a8d98ac6789a","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":2026,"response_code":200}
{"msg":"path is \"/data/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.687545329Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"datafile"}
{"msg":"dataset is oxp_0a6d51c0-d0df-4812-9c5a-213b208d92aa/crucible/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.699021319Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"datafile"}
{"msg":"region 6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.708911654Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.709124459Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.709175423Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"uri":"/crucible/0/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","method":"DELETE","req_id":"81f9b703-5a4a-48b1-b516-ed8fec6cd4b7","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":21699,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.710413008Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.714748826Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"uri":"/crucible/0/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","method":"GET","req_id":"2da09df4-7bb6-4804-b76f-0f0d7dc2059e","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":36,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.715396014Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0a6d51c0-d0df-4812-9c5a-213b208d92aa/crucible/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.726967944Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"region":"6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.799576894Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"uri":"/crucible/0/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","method":"GET","req_id":"52bcaa3b-6735-447e-aca0-2fbc7a85da36","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":81,"response_code":200}
{"msg":"region 6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.882067015Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-10-03T21:07:25.893575346Z","hostname":"oxz_crucible_5d6c3917-53a5-4e6c-b4da-7d1d3b5a420c","pid":25195,"uri":"/crucible/0/regions/6cb64b64-2ca1-46a2-9fec-1db6ffb75e7a","method":"GET","req_id":"7464dcc4-ac39-405d-84df-fa671dd0b6e5","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":41,"response_code":200}