[ Mar 30 23:26:37 Enabled. ]
[ Mar 30 23:26:37 Rereading configuration. ]
[ Mar 30 23:26:38 Rereading configuration. ]
[ Mar 30 23:26:41 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6797e705-a9d8-4257-8abd-3a6b647ed68b/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:26:41.525916638Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115}
{"msg":"dataset: \"oxp_6797e705-a9d8-4257-8abd-3a6b647ed68b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:26:41.526918867Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:26:41.526933547Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:26:41.526941187Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:26:41.54491556Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:26:41.634226256Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.023254737Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33416"}
{"msg":"region a568476c-d977-461a-b6af-9a9f43cb1367 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.04990123Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.052902036Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.052944357Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"uri":"/crucible/0/regions","method":"POST","req_id":"eb042984-f8fb-4ff7-b2da-a11cfe3a2c13","remote_addr":"[fd00:1122:3344:101::a]:33416","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":14591,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_6797e705-a9d8-4257-8abd-3a6b647ed68b/crucible/regions/a568476c-d977-461a-b6af-9a9f43cb1367","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.063280488Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_6797e705-a9d8-4257-8abd-3a6b647ed68b/crucible/regions/a568476c-d977-461a-b6af-9a9f43cb1367","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.063297628Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a568476c-d977-461a-b6af-9a9f43cb1367\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a568476c-d977-461a-b6af-9a9f43cb1367\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.149596167Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"region":"a568476c-d977-461a-b6af-9a9f43cb1367","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a568476c-d977-461a-b6af-9a9f43cb1367\" \"--data\" \"/data/regions/a568476c-d977-461a-b6af-9a9f43cb1367\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.151610645Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"region":"a568476c-d977-461a-b6af-9a9f43cb1367","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.317163903Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"uri":"/crucible/0/regions","method":"POST","req_id":"a9f99998-cf6c-4e3a-bd9a-0da6c3b8771a","remote_addr":"[fd00:1122:3344:101::a]:33416","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.546930652Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"region":"a568476c-d977-461a-b6af-9a9f43cb1367","component":"worker"}
{"msg":"region a568476c-d977-461a-b6af-9a9f43cb1367 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.547005033Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.547214337Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a568476c-d977-461a-b6af-9a9f43cb1367","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.547556243Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a568476c-d977-461a-b6af-9a9f43cb1367","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.557781573Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.583887687Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a568476c-d977-461a-b6af-9a9f43cb1367","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.591116541Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a568476c-d977-461a-b6af-9a9f43cb1367","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.591833064Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.591892735Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.591944626Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.591980306Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.597795254Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:22.60348881Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:23.050179149Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"uri":"/crucible/0/regions","method":"POST","req_id":"a78bdbc6-09ef-4e86-8811-e0de0634c856","remote_addr":"[fd00:1122:3344:101::a]:33416","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":59,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.223871802Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"uri":"/crucible/0/regions/a568476c-d977-461a-b6af-9a9f43cb1367","method":"GET","req_id":"447f20c9-460d-47b1-8a49-890a0abae1cc","remote_addr":"[fd00:1122:3344:101::a]:33416","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":95,"response_code":200}
{"msg":"path is \"/data/regions/a568476c-d977-461a-b6af-9a9f43cb1367\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.226722885Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"datafile"}
{"msg":"dataset is oxp_6797e705-a9d8-4257-8abd-3a6b647ed68b/crucible/regions/a568476c-d977-461a-b6af-9a9f43cb1367","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.236515537Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"datafile"}
{"msg":"region a568476c-d977-461a-b6af-9a9f43cb1367 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.245053095Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.245299749Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.245371441Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"uri":"/crucible/0/regions/a568476c-d977-461a-b6af-9a9f43cb1367","method":"DELETE","req_id":"90bcdd90-a3cb-4c9e-8e81-970ed08a6cbf","remote_addr":"[fd00:1122:3344:101::a]:33416","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":18602,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-a568476c-d977-461a-b6af-9a9f43cb1367 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.246696135Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.250521106Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"uri":"/crucible/0/regions/a568476c-d977-461a-b6af-9a9f43cb1367","method":"GET","req_id":"a766ac93-b2be-4c19-827a-de9c18f9fa1f","remote_addr":"[fd00:1122:3344:101::a]:33416","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":65,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.256050729Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6797e705-a9d8-4257-8abd-3a6b647ed68b/crucible/regions/a568476c-d977-461a-b6af-9a9f43cb1367\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.265494604Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"region":"a568476c-d977-461a-b6af-9a9f43cb1367","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.353630387Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"uri":"/crucible/0/regions/a568476c-d977-461a-b6af-9a9f43cb1367","method":"GET","req_id":"8b499c75-30a2-441f-89af-2549966d216a","remote_addr":"[fd00:1122:3344:101::a]:33416","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"region a568476c-d977-461a-b6af-9a9f43cb1367 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.385672171Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-30T23:27:24.505262828Z","hostname":"oxz_crucible_1fa0b4c5-5c40-48d0-8bf1-6b56320abc81","pid":22115,"uri":"/crucible/0/regions/a568476c-d977-461a-b6af-9a9f43cb1367","method":"GET","req_id":"3da9199e-4462-42be-a0af-8f16c7014cc8","remote_addr":"[fd00:1122:3344:101::a]:33416","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":65,"response_code":200}