[ Aug 31 21:31:50 Enabled. ]
[ Aug 31 21:31:50 Rereading configuration. ]
[ Aug 31 21:31:52 Rereading configuration. ]
[ Aug 31 21:31:55 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ad8a31742adc45e925e63443a5b43c8e30604022\nCommit timestamp: 2026-07-28T19:41:59.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:31:55.965732037Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749}
{"msg":"dataset: \"oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:31:55.966495585Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:31:55.966510085Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:31:55.966517345Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:31:55.990173504Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:31:56.069055661Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:35.746516798Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:36909"}
{"msg":"region f0f2187d-83fb-4026-8a35-a216b65cd9d4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:35.781567192Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:35.78630272Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:35.790772565Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"uri":"/crucible/0/regions","method":"POST","req_id":"cdcf9f99-aa01-412b-8b85-4ddca5544dc1","remote_addr":"[fd00:1122:3344:101::b]:36909","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":15750,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:35.793771746Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:35.793793356Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f0f2187d-83fb-4026-8a35-a216b65cd9d4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:35.860920974Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"region":"f0f2187d-83fb-4026-8a35-a216b65cd9d4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f0f2187d-83fb-4026-8a35-a216b65cd9d4\" \"--data\" \"/data/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:35.862894334Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"region":"f0f2187d-83fb-4026-8a35-a216b65cd9d4","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:35.999600385Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"region":"f0f2187d-83fb-4026-8a35-a216b65cd9d4","component":"worker"}
{"msg":"region f0f2187d-83fb-4026-8a35-a216b65cd9d4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.003506805Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.003804508Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.005412154Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"uri":"/crucible/0/regions","method":"POST","req_id":"348029fa-cf12-4fa7-9038-b8a827604de3","remote_addr":"[fd00:1122:3344:101::b]:36909","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"creating missing downstairs instance downstairs-f0f2187d-83fb-4026-8a35-a216b65cd9d4","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.006373014Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f0f2187d-83fb-4026-8a35-a216b65cd9d4","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.013631967Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.024860171Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f0f2187d-83fb-4026-8a35-a216b65cd9d4","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.026526717Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.027247975Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.027300785Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.027345756Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.027375786Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.030207614Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:32:36.033301496Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.706167076Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:39846"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.706890483Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"uri":"/crucible/0/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4","method":"GET","req_id":"7f18f8f9-099a-479a-a670-2cd32390b057","remote_addr":"[fd00:1122:3344:101::b]:39846","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"path is \"/data/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.709787262Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"dataset is oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.716812473Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"snapshot is oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4@4411979b-e4a3-4a97-99d5-00ef25fe1742","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.723729393Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4\", \"4411979b-e4a3-4a97-99d5-00ef25fe1742\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.723758234Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.736708054Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"uri":"/crucible/0/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4/snapshots/4411979b-e4a3-4a97-99d5-00ef25fe1742","method":"GET","req_id":"439daab9-eeba-49a9-a85d-8a7ce30dad7c","remote_addr":"[fd00:1122:3344:101::b]:39846","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26984,"response_code":200}
{"msg":"path is \"/data/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.748376472Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"dataset is oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.754890858Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"snapshot is oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4@4411979b-e4a3-4a97-99d5-00ef25fe1742","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.761210332Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4\", \"4411979b-e4a3-4a97-99d5-00ef25fe1742\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.761229222Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot f0f2187d-83fb-4026-8a35-a216b65cd9d4-4411979b-e4a3-4a97-99d5-00ef25fe1742 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.769745558Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"applying SMF actions for region f0f2187d-83fb-4026-8a35-a216b65cd9d4 running snapshot 4411979b-e4a3-4a97-99d5-00ef25fe1742 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.775253274Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.775274234Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"uri":"/crucible/0/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4/snapshots/4411979b-e4a3-4a97-99d5-00ef25fe1742/run","method":"POST","req_id":"00642f78-08c2-424e-bcf8-2cb803ad4e2f","remote_addr":"[fd00:1122:3344:101::b]:39846","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26926,"response_code":200}
{"msg":"creating missing snapshot instance snapshot-f0f2187d-83fb-4026-8a35-a216b65cd9d4-4411979b-e4a3-4a97-99d5-00ef25fe1742","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.791131374Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-f0f2187d-83fb-4026-8a35-a216b65cd9d4-4411979b-e4a3-4a97-99d5-00ef25fe1742","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.794329596Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-f0f2187d-83fb-4026-8a35-a216b65cd9d4-4411979b-e4a3-4a97-99d5-00ef25fe1742","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.802753792Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4/.zfs/snapshot/4411979b-e4a3-4a97-99d5-00ef25fe1742","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.803090215Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.803140356Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.803187846Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.803242657Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.803309007Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.805956184Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.808197027Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"worker"}
{"msg":"region f0f2187d-83fb-4026-8a35-a216b65cd9d4 running snapshot 4411979b-e4a3-4a97-99d5-00ef25fe1742 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.808211367Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"path is \"/data/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4\"","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.909380729Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"dataset is oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.915818784Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"datafile"}
{"msg":"snapshot is oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4@4411979b-e4a3-4a97-99d5-00ef25fe1742","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.921158568Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8632448a-efc2-498f-8b8e-3fc4e788e7d6/crucible/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4\", \"4411979b-e4a3-4a97-99d5-00ef25fe1742\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.921202989Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-08-31T21:43:25.927515532Z","hostname":"oxz_crucible_948211a2-26ca-45b1-9fd7-f9bff781dc00","pid":25749,"uri":"/crucible/0/regions/f0f2187d-83fb-4026-8a35-a216b65cd9d4/snapshots/4411979b-e4a3-4a97-99d5-00ef25fe1742/run","method":"POST","req_id":"8483c34b-f70b-43e0-8a27-f9f79e8f9a31","remote_addr":"[fd00:1122:3344:101::b]:39846","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18165,"response_code":200}