[ Jun 22 21:13:12 Enabled. ]
[ Jun 22 21:13:12 Rereading configuration. ]
[ Jun 22 21:13:15 Rereading configuration. ]
[ Jun 22 21:13:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9609a0c3-04b9-4070-8a24-714d73908dac/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: bd9a0e2abe6b6b89aec8c85f4ee57474144ed150\nCommit timestamp: 2026-05-14T15:48:00.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:20.612741376Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169}
{"msg":"dataset: \"oxp_9609a0c3-04b9-4070-8a24-714d73908dac/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:20.612947968Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:20.612958908Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:20.612964548Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:20.632381919Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:20.69989941Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.13094333Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:34819"}
{"msg":"region 83cf7754-20d6-422e-82fe-c71999e50d17 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.163731322Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.169147241Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.170114008Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"uri":"/crucible/0/regions","method":"POST","req_id":"c1bc4827-df3a-4e5e-b42f-3c1b79bc6b2d","remote_addr":"[fd00:1122:3344:101::c]:34819","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":14816,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_9609a0c3-04b9-4070-8a24-714d73908dac/crucible/regions/83cf7754-20d6-422e-82fe-c71999e50d17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.177308411Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9609a0c3-04b9-4070-8a24-714d73908dac/crucible/regions/83cf7754-20d6-422e-82fe-c71999e50d17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.177330581Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"83cf7754-20d6-422e-82fe-c71999e50d17\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/83cf7754-20d6-422e-82fe-c71999e50d17\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.33112556Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"region":"83cf7754-20d6-422e-82fe-c71999e50d17","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"83cf7754-20d6-422e-82fe-c71999e50d17\" \"--data\" \"/data/regions/83cf7754-20d6-422e-82fe-c71999e50d17\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.333370997Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"region":"83cf7754-20d6-422e-82fe-c71999e50d17","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.409279859Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"uri":"/crucible/0/regions","method":"POST","req_id":"4b7d5083-28c2-46bb-bd6f-5e606bdeb63c","remote_addr":"[fd00:1122:3344:101::c]:34819","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.573333893Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"region":"83cf7754-20d6-422e-82fe-c71999e50d17","component":"worker"}
{"msg":"region 83cf7754-20d6-422e-82fe-c71999e50d17 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.573424564Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.573633285Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-83cf7754-20d6-422e-82fe-c71999e50d17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.575905542Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-83cf7754-20d6-422e-82fe-c71999e50d17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.588495944Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.609297885Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-83cf7754-20d6-422e-82fe-c71999e50d17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.612658329Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/83cf7754-20d6-422e-82fe-c71999e50d17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.613399655Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.613452835Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.613495255Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.613541406Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.631792809Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.647360232Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:51.970921428Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"uri":"/crucible/0/regions","method":"POST","req_id":"ccf16790-dd2e-4521-9f26-d32e562e773d","remote_addr":"[fd00:1122:3344:101::c]:34819","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":79,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.563992855Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"uri":"/crucible/0/regions/83cf7754-20d6-422e-82fe-c71999e50d17","method":"GET","req_id":"41e40693-27ee-459e-a51a-32346a26a8a0","remote_addr":"[fd00:1122:3344:101::c]:34819","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"path is \"/data/regions/83cf7754-20d6-422e-82fe-c71999e50d17\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.567085317Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"datafile"}
{"msg":"dataset is oxp_9609a0c3-04b9-4070-8a24-714d73908dac/crucible/regions/83cf7754-20d6-422e-82fe-c71999e50d17","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.576237604Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"datafile"}
{"msg":"region 83cf7754-20d6-422e-82fe-c71999e50d17 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.584573835Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.584805656Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.584872017Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"uri":"/crucible/0/regions/83cf7754-20d6-422e-82fe-c71999e50d17","method":"DELETE","req_id":"e63534c0-2a0a-4549-b62e-5563ab42c924","remote_addr":"[fd00:1122:3344:101::c]:34819","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17755,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-83cf7754-20d6-422e-82fe-c71999e50d17 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.585982305Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.588882886Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"uri":"/crucible/0/regions/83cf7754-20d6-422e-82fe-c71999e50d17","method":"GET","req_id":"92710a77-e60f-46b1-99d0-028dc43db477","remote_addr":"[fd00:1122:3344:101::c]:34819","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":45,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.596063948Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9609a0c3-04b9-4070-8a24-714d73908dac/crucible/regions/83cf7754-20d6-422e-82fe-c71999e50d17\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.605361976Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"region":"83cf7754-20d6-422e-82fe-c71999e50d17","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.684305621Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"uri":"/crucible/0/regions/83cf7754-20d6-422e-82fe-c71999e50d17","method":"GET","req_id":"2d9a8b1e-c446-43cb-bb2f-65aca614cc37","remote_addr":"[fd00:1122:3344:101::c]:34819","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":37,"response_code":200}
{"msg":"region 83cf7754-20d6-422e-82fe-c71999e50d17 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.736996944Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-22T21:13:52.774556558Z","hostname":"oxz_crucible_bbc7a858-83d9-4409-9e87-0704c22eb5fb","pid":24169,"uri":"/crucible/0/regions/83cf7754-20d6-422e-82fe-c71999e50d17","method":"GET","req_id":"1d45f4eb-b3ff-451c-b9cf-ddb9370912c5","remote_addr":"[fd00:1122:3344:101::c]:34819","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":51,"response_code":200}