[ Apr  7 18:45:13 Enabled. ]
[ Apr  7 18:45:13 Rereading configuration. ]
[ Apr  7 18:45:15 Rereading configuration. ]
[ Apr  7 18:45:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:22.376281115Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734}
{"msg":"dataset: \"oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:22.377171359Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:22.377182535Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:22.377189266Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:22.388784341Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:22.689460495Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:55.726071507Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:43677"}
{"msg":"region 5834b167-c0bb-4448-ab54-50f1aff243b3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:55.765877806Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:55.768558285Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:55.768576156Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions","method":"POST","req_id":"739132c8-2345-44d5-ad66-6bf948ce8bc8","remote_addr":"[fd00:1122:3344:101::c]:43677","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18772,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:55.774967589Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:55.77498021Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5834b167-c0bb-4448-ab54-50f1aff243b3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5834b167-c0bb-4448-ab54-50f1aff243b3\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:55.831991001Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"region":"5834b167-c0bb-4448-ab54-50f1aff243b3","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5834b167-c0bb-4448-ab54-50f1aff243b3\" \"--data\" \"/data/regions/5834b167-c0bb-4448-ab54-50f1aff243b3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:55.834288889Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"region":"5834b167-c0bb-4448-ab54-50f1aff243b3","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:55.932481396Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions","method":"POST","req_id":"89e7a20f-d83b-422a-95fb-4376c4174ae3","remote_addr":"[fd00:1122:3344:101::c]:43677","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.184885523Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"region":"5834b167-c0bb-4448-ab54-50f1aff243b3","component":"worker"}
{"msg":"region 5834b167-c0bb-4448-ab54-50f1aff243b3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.184925524Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.185013966Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5834b167-c0bb-4448-ab54-50f1aff243b3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.185394966Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5834b167-c0bb-4448-ab54-50f1aff243b3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.195677449Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.212053109Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5834b167-c0bb-4448-ab54-50f1aff243b3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.215021105Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5834b167-c0bb-4448-ab54-50f1aff243b3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.215737733Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.215798835Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.215838526Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.215873807Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.219878789Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.224255121Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:45:56.309930247Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions","method":"POST","req_id":"9b45776b-efba-40f8-a816-4582aac1e312","remote_addr":"[fd00:1122:3344:101::c]:43677","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.008446011Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:40355"}
{"msg":"region ee589c8a-87a5-4522-ab3a-07d49dea1522 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.009059827Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.009401636Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.009409206Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions","method":"POST","req_id":"7d802988-0826-4d57-9f25-78be66bc996c","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":909,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/ee589c8a-87a5-4522-ab3a-07d49dea1522","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.016337933Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/ee589c8a-87a5-4522-ab3a-07d49dea1522","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.016358784Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ee589c8a-87a5-4522-ab3a-07d49dea1522\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ee589c8a-87a5-4522-ab3a-07d49dea1522\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.073156749Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"region":"ee589c8a-87a5-4522-ab3a-07d49dea1522","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ee589c8a-87a5-4522-ab3a-07d49dea1522\" \"--data\" \"/data/regions/ee589c8a-87a5-4522-ab3a-07d49dea1522\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.07319679Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"region":"ee589c8a-87a5-4522-ab3a-07d49dea1522","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.168186494Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"region":"ee589c8a-87a5-4522-ab3a-07d49dea1522","component":"worker"}
{"msg":"region ee589c8a-87a5-4522-ab3a-07d49dea1522 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.168217245Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.168300667Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ee589c8a-87a5-4522-ab3a-07d49dea1522","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.189314205Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ee589c8a-87a5-4522-ab3a-07d49dea1522","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.195676408Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.204929485Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ee589c8a-87a5-4522-ab3a-07d49dea1522","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.207906331Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ee589c8a-87a5-4522-ab3a-07d49dea1522","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.208184878Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.20823328Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.208276881Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.208316412Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.211946595Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.216665776Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:34.3753011Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions","method":"POST","req_id":"05fbf9f5-b598-4ee1-89d5-9077074a6a7f","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.352270311Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions/5834b167-c0bb-4448-ab54-50f1aff243b3","method":"GET","req_id":"7b8fa3cf-165a-48c6-9309-6ad38a703311","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":80,"response_code":200}
{"msg":"path is \"/data/regions/5834b167-c0bb-4448-ab54-50f1aff243b3\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.356291544Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"dataset is oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.364130715Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"snapshot is oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3@9da45c1b-a6ca-4fd7-9867-89ef4408d06b","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.372463808Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3\", \"9da45c1b-a6ca-4fd7-9867-89ef4408d06b\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.372481359Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.383548963Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions/5834b167-c0bb-4448-ab54-50f1aff243b3/snapshots/9da45c1b-a6ca-4fd7-9867-89ef4408d06b","method":"GET","req_id":"e7aff79a-420b-43c3-8341-96116b5ef894","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":27276,"response_code":200}
{"msg":"path is \"/data/regions/5834b167-c0bb-4448-ab54-50f1aff243b3\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.388649873Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"dataset is oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.39751334Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"snapshot is oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3@9da45c1b-a6ca-4fd7-9867-89ef4408d06b","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.406352007Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3\", \"9da45c1b-a6ca-4fd7-9867-89ef4408d06b\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.406387318Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 5834b167-c0bb-4448-ab54-50f1aff243b3-9da45c1b-a6ca-4fd7-9867-89ef4408d06b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.416471266Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"applying SMF actions for region 5834b167-c0bb-4448-ab54-50f1aff243b3 running snapshot 9da45c1b-a6ca-4fd7-9867-89ef4408d06b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.416699352Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.416719482Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions/5834b167-c0bb-4448-ab54-50f1aff243b3/snapshots/9da45c1b-a6ca-4fd7-9867-89ef4408d06b/run","method":"POST","req_id":"7e409f97-ac9c-4780-9622-bd57f164e053","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":28083,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-5834b167-c0bb-4448-ab54-50f1aff243b3-9da45c1b-a6ca-4fd7-9867-89ef4408d06b","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.423181918Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-5834b167-c0bb-4448-ab54-50f1aff243b3-9da45c1b-a6ca-4fd7-9867-89ef4408d06b","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.440518382Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-5834b167-c0bb-4448-ab54-50f1aff243b3-9da45c1b-a6ca-4fd7-9867-89ef4408d06b","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.452785277Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5834b167-c0bb-4448-ab54-50f1aff243b3/.zfs/snapshot/9da45c1b-a6ca-4fd7-9867-89ef4408d06b","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.453111525Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.453172336Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.453231458Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.453284579Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.453351051Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.458926702Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.464712282Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"region 5834b167-c0bb-4448-ab54-50f1aff243b3 running snapshot 9da45c1b-a6ca-4fd7-9867-89ef4408d06b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.464733183Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"path is \"/data/regions/5834b167-c0bb-4448-ab54-50f1aff243b3\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.734362251Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"dataset is oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.741037112Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"snapshot is oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3@9da45c1b-a6ca-4fd7-9867-89ef4408d06b","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.746592064Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/5834b167-c0bb-4448-ab54-50f1aff243b3\", \"9da45c1b-a6ca-4fd7-9867-89ef4408d06b\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.746624565Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:35.751270344Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions/5834b167-c0bb-4448-ab54-50f1aff243b3/snapshots/9da45c1b-a6ca-4fd7-9867-89ef4408d06b/run","method":"POST","req_id":"92152d54-8d2f-4e3b-983a-fe33142c3ddb","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":16940,"response_code":200}
{"msg":"region 4c61fa17-adb5-4051-9e6c-4526b8375ed1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.486874417Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.487052561Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.487078022Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions","method":"POST","req_id":"6ee5c442-fbfd-440e-8018-cea39714f8a8","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":208,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.493608899Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.49363534Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4c61fa17-adb5-4051-9e6c-4526b8375ed1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.550891237Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"region":"4c61fa17-adb5-4051-9e6c-4526b8375ed1","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4c61fa17-adb5-4051-9e6c-4526b8375ed1\" \"--data\" \"/data/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.550929128Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"region":"4c61fa17-adb5-4051-9e6c-4526b8375ed1","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.637396203Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"region":"4c61fa17-adb5-4051-9e6c-4526b8375ed1","component":"worker"}
{"msg":"region 4c61fa17-adb5-4051-9e6c-4526b8375ed1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.637424114Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.637547657Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4c61fa17-adb5-4051-9e6c-4526b8375ed1","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.637932037Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4c61fa17-adb5-4051-9e6c-4526b8375ed1","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.64078538Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.647713638Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4c61fa17-adb5-4051-9e6c-4526b8375ed1","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.651195547Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.651515965Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.651557086Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.651591857Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.651629178Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.655480667Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.666014596Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:56:41.794755855Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions","method":"POST","req_id":"8c5dd6ab-a8bf-4e02-8fb6-fafaf04d6cd3","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":55,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:07.981947092Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1","method":"GET","req_id":"8dc98cfe-b86f-4437-b097-c7ea1ad1ac40","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":77,"response_code":200}
{"msg":"path is \"/data/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:07.985785351Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"dataset is oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:07.995609013Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"region 4c61fa17-adb5-4051-9e6c-4526b8375ed1 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:08.002472778Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:08.003098904Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:08.003118855Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1","method":"DELETE","req_id":"a4395d96-ff04-40ac-9ade-e3865f9b73a1","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":17364,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-4c61fa17-adb5-4051-9e6c-4526b8375ed1 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:08.003664689Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:08.006346937Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1","method":"GET","req_id":"17827b63-3e38-4f60-af79-1449e9c6f402","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":29,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:08.025071327Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d67a399d-534d-42b0-aec9-f16b877e3b3c/crucible/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1\"","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:08.032766685Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"region":"4c61fa17-adb5-4051-9e6c-4526b8375ed1","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:08.102593354Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1","method":"GET","req_id":"e1e27725-a308-44de-92f8-1c9720cad4ae","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":66,"response_code":200}
{"msg":"region 4c61fa17-adb5-4051-9e6c-4526b8375ed1 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:08.128011955Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-04-07T18:57:08.25933179Z","hostname":"oxz_crucible_1db1a100-53d3-4aa6-9e15-6a20d8cfd2c4","pid":29734,"uri":"/crucible/0/regions/4c61fa17-adb5-4051-9e6c-4526b8375ed1","method":"GET","req_id":"04db0bc4-02f6-4a09-97ab-e9d38f675c05","remote_addr":"[fd00:1122:3344:101::c]:40355","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":33,"response_code":200}