[ Apr 10 03:13:36 Enabled. ]
[ Apr 10 03:13:36 Rereading configuration. ]
[ Apr 10 03:13:39 Rereading configuration. ]
[ Apr 10 03:13:44 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_71213a5d-546a-4000-9a1b-a22b014f4756/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:13:44.215965948Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863}
{"msg":"dataset: \"oxp_71213a5d-546a-4000-9a1b-a22b014f4756/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:13:44.216760964Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:13:44.216891286Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:13:44.216898497Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:13:44.231962205Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:13:44.298329023Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.401020917Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57448"}
{"msg":"region 1020c2c5-70ea-44af-b2e9-45a632f15177 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.425959663Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.429123946Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.429176617Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"uri":"/crucible/0/regions","method":"POST","req_id":"6db1bf63-49dd-4cf9-abd2-4fe946580891","remote_addr":"[fd00:1122:3344:101::a]:57448","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":14164,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_71213a5d-546a-4000-9a1b-a22b014f4756/crucible/regions/1020c2c5-70ea-44af-b2e9-45a632f15177","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.438195506Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_71213a5d-546a-4000-9a1b-a22b014f4756/crucible/regions/1020c2c5-70ea-44af-b2e9-45a632f15177","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.438215706Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1020c2c5-70ea-44af-b2e9-45a632f15177\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1020c2c5-70ea-44af-b2e9-45a632f15177\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.495667927Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"region":"1020c2c5-70ea-44af-b2e9-45a632f15177","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1020c2c5-70ea-44af-b2e9-45a632f15177\" \"--data\" \"/data/regions/1020c2c5-70ea-44af-b2e9-45a632f15177\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.497778189Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"region":"1020c2c5-70ea-44af-b2e9-45a632f15177","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.674162121Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"uri":"/crucible/0/regions","method":"POST","req_id":"04442203-686c-4469-9520-d17ee4e023d9","remote_addr":"[fd00:1122:3344:101::a]:57448","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.847539634Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"region":"1020c2c5-70ea-44af-b2e9-45a632f15177","component":"worker"}
{"msg":"region 1020c2c5-70ea-44af-b2e9-45a632f15177 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.847609545Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.84782818Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1020c2c5-70ea-44af-b2e9-45a632f15177","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.848194577Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1020c2c5-70ea-44af-b2e9-45a632f15177","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.85388622Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.868283836Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1020c2c5-70ea-44af-b2e9-45a632f15177","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.871008327Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1020c2c5-70ea-44af-b2e9-45a632f15177","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.871644622Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.871706854Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.871747775Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.871782015Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.877528529Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:06.881759323Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.241505987Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"uri":"/crucible/0/regions","method":"POST","req_id":"75c70040-1e50-48be-960d-82d192ee61dd","remote_addr":"[fd00:1122:3344:101::a]:57448","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.582975257Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"uri":"/crucible/0/regions/1020c2c5-70ea-44af-b2e9-45a632f15177","method":"GET","req_id":"5d9d460d-d815-4d18-ab3e-5f674650d332","remote_addr":"[fd00:1122:3344:101::a]:57448","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":78,"response_code":200}
{"msg":"path is \"/data/regions/1020c2c5-70ea-44af-b2e9-45a632f15177\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.586319924Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"datafile"}
{"msg":"dataset is oxp_71213a5d-546a-4000-9a1b-a22b014f4756/crucible/regions/1020c2c5-70ea-44af-b2e9-45a632f15177","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.594586248Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"datafile"}
{"msg":"region 1020c2c5-70ea-44af-b2e9-45a632f15177 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.6022505Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.602429214Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"uri":"/crucible/0/regions/1020c2c5-70ea-44af-b2e9-45a632f15177","method":"DELETE","req_id":"e1a20ce2-8268-4765-9f60-1823a01f43af","remote_addr":"[fd00:1122:3344:101::a]:57448","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":16135,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.602479185Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-1020c2c5-70ea-44af-b2e9-45a632f15177 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.603411463Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.604815751Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"uri":"/crucible/0/regions/1020c2c5-70ea-44af-b2e9-45a632f15177","method":"GET","req_id":"2a2a1c4a-2079-4c6f-8aea-17826729ee5d","remote_addr":"[fd00:1122:3344:101::a]:57448","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":31,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.607553115Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_71213a5d-546a-4000-9a1b-a22b014f4756/crucible/regions/1020c2c5-70ea-44af-b2e9-45a632f15177\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.616190047Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"region":"1020c2c5-70ea-44af-b2e9-45a632f15177","component":"worker"}
{"msg":"region 1020c2c5-70ea-44af-b2e9-45a632f15177 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.690795778Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-10T03:14:07.747028615Z","hostname":"oxz_crucible_1da357ce-9acf-4c0f-9b63-3f1344246adf","pid":25863,"uri":"/crucible/0/regions/1020c2c5-70ea-44af-b2e9-45a632f15177","method":"GET","req_id":"babd1a31-5d54-45c2-a25f-8f423bb97b40","remote_addr":"[fd00:1122:3344:101::a]:57448","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":57,"response_code":200}