[ Nov 12 20:45:18 Enabled. ]
[ Nov 12 20:45:18 Rereading configuration. ]
[ Nov 12 20:45:21 Rereading configuration. ]
[ Nov 12 20:45:24 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:45:24.78422121Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694}
{"msg":"dataset: \"oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:45:24.78499441Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:45:24.785005721Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:45:24.785011081Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:45:24.797113474Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:45:24.880708487Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.166932632Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44088"}
{"msg":"region b19f679b-f10c-49dd-afca-9806ccddd7d4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.207209854Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.210410537Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.210440807Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions","method":"POST","req_id":"f6389efc-fa2d-4984-82b8-8fead7ec6c07","remote_addr":"[fd00:1122:3344:101::a]:44088","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19387,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.217347416Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.217364189Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b19f679b-f10c-49dd-afca-9806ccddd7d4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.280799158Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"region":"b19f679b-f10c-49dd-afca-9806ccddd7d4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b19f679b-f10c-49dd-afca-9806ccddd7d4\" \"--data\" \"/data/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.283400726Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"region":"b19f679b-f10c-49dd-afca-9806ccddd7d4","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.519510587Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions","method":"POST","req_id":"1de8f2e2-dae9-4a35-afa8-e33177962e5e","remote_addr":"[fd00:1122:3344:101::a]:44088","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.642773275Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"region":"b19f679b-f10c-49dd-afca-9806ccddd7d4","component":"worker"}
{"msg":"region b19f679b-f10c-49dd-afca-9806ccddd7d4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.642823066Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.643003779Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b19f679b-f10c-49dd-afca-9806ccddd7d4","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.64348637Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b19f679b-f10c-49dd-afca-9806ccddd7d4","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.651628414Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.667967777Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b19f679b-f10c-49dd-afca-9806ccddd7d4","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.670770219Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.671405116Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.671470546Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.671511066Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.671545435Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.675146493Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.678777186Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:46:19.99927437Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions","method":"POST","req_id":"1d5e6164-45f0-4e5e-81d4-a162f4718d6c","remote_addr":"[fd00:1122:3344:101::a]:44088","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.203597268Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62838"}
{"msg":"region bd0e6fca-096a-4ca4-8456-febbff3e81f9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.204300026Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.204613824Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.204632925Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions","method":"POST","req_id":"26946f6f-7dfa-4bf4-a030-35bb9c7770e8","remote_addr":"[fd00:1122:3344:101::a]:62838","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":961,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/bd0e6fca-096a-4ca4-8456-febbff3e81f9","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.214377597Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/bd0e6fca-096a-4ca4-8456-febbff3e81f9","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.214401597Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"bd0e6fca-096a-4ca4-8456-febbff3e81f9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bd0e6fca-096a-4ca4-8456-febbff3e81f9\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.276102354Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"region":"bd0e6fca-096a-4ca4-8456-febbff3e81f9","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bd0e6fca-096a-4ca4-8456-febbff3e81f9\" \"--data\" \"/data/regions/bd0e6fca-096a-4ca4-8456-febbff3e81f9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.276143065Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"region":"bd0e6fca-096a-4ca4-8456-febbff3e81f9","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.398298637Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"region":"bd0e6fca-096a-4ca4-8456-febbff3e81f9","component":"worker"}
{"msg":"region bd0e6fca-096a-4ca4-8456-febbff3e81f9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.398335927Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.39844429Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-bd0e6fca-096a-4ca4-8456-febbff3e81f9","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.414821434Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bd0e6fca-096a-4ca4-8456-febbff3e81f9","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.422851792Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.435325255Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bd0e6fca-096a-4ca4-8456-febbff3e81f9","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.439812001Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bd0e6fca-096a-4ca4-8456-febbff3e81f9","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.440069718Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.440121919Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.44016149Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.440199731Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.444488892Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.449030779Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:20.524464462Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions","method":"POST","req_id":"4cfcd4fa-8f32-4096-b871-7ad9f2183ad9","remote_addr":"[fd00:1122:3344:101::a]:62838","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":80,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.6870765Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4","method":"GET","req_id":"a8898933-dfec-49f6-9f3a-0ead7926b4be","remote_addr":"[fd00:1122:3344:101::a]:62838","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":71,"response_code":200}
{"msg":"path is \"/data/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.690681103Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"dataset is oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.696579986Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"snapshot is oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4@1251cdc9-0557-46ad-b21c-2182ec9fe215","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.703587537Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4\", \"1251cdc9-0557-46ad-b21c-2182ec9fe215\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.703600607Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.71143938Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4/snapshots/1251cdc9-0557-46ad-b21c-2182ec9fe215","method":"GET","req_id":"ba4d7ccd-c65e-4a83-af53-75af26dd3e6b","remote_addr":"[fd00:1122:3344:101::a]:62838","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20785,"response_code":200}
{"msg":"path is \"/data/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.714386037Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"dataset is oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.720730581Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"snapshot is oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4@1251cdc9-0557-46ad-b21c-2182ec9fe215","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.726247153Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4\", \"1251cdc9-0557-46ad-b21c-2182ec9fe215\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.726266904Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot b19f679b-f10c-49dd-afca-9806ccddd7d4-1251cdc9-0557-46ad-b21c-2182ec9fe215 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.733556182Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.733685926Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4/snapshots/1251cdc9-0557-46ad-b21c-2182ec9fe215/run","method":"POST","req_id":"77d39aff-cd63-46dd-bc01-3b7932054175","remote_addr":"[fd00:1122:3344:101::a]:62838","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19325,"response_code":200}
{"msg":"applying SMF actions for region b19f679b-f10c-49dd-afca-9806ccddd7d4 running snapshot 1251cdc9-0557-46ad-b21c-2182ec9fe215 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.733704766Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-b19f679b-f10c-49dd-afca-9806ccddd7d4-1251cdc9-0557-46ad-b21c-2182ec9fe215","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.737903405Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-b19f679b-f10c-49dd-afca-9806ccddd7d4-1251cdc9-0557-46ad-b21c-2182ec9fe215","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.745751888Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-b19f679b-f10c-49dd-afca-9806ccddd7d4-1251cdc9-0557-46ad-b21c-2182ec9fe215","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.756478656Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4/.zfs/snapshot/1251cdc9-0557-46ad-b21c-2182ec9fe215","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.756742083Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.756787844Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.756828885Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.756867666Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.756927727Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.761062174Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.765822088Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"region b19f679b-f10c-49dd-afca-9806ccddd7d4 running snapshot 1251cdc9-0557-46ad-b21c-2182ec9fe215 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.765839048Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"path is \"/data/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.868451944Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"dataset is oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.875055545Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"snapshot is oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4@1251cdc9-0557-46ad-b21c-2182ec9fe215","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.880885906Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4\", \"1251cdc9-0557-46ad-b21c-2182ec9fe215\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.880917466Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:21.886698106Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions/b19f679b-f10c-49dd-afca-9806ccddd7d4/snapshots/1251cdc9-0557-46ad-b21c-2182ec9fe215/run","method":"POST","req_id":"25dcd8fc-0262-4fc0-9137-bb319012cf2b","remote_addr":"[fd00:1122:3344:101::a]:62838","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":18273,"response_code":200}
{"msg":"region 306b53c8-7332-47ac-a835-d7254325a814 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.522577452Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.522738516Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.522789887Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions","method":"POST","req_id":"784bf8e2-a0b1-4920-8050-51aa42a6dbfc","remote_addr":"[fd00:1122:3344:101::a]:62838","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":211,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/306b53c8-7332-47ac-a835-d7254325a814","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.528193117Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/306b53c8-7332-47ac-a835-d7254325a814","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.528241188Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"306b53c8-7332-47ac-a835-d7254325a814\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/306b53c8-7332-47ac-a835-d7254325a814\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.573702555Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"region":"306b53c8-7332-47ac-a835-d7254325a814","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"306b53c8-7332-47ac-a835-d7254325a814\" \"--data\" \"/data/regions/306b53c8-7332-47ac-a835-d7254325a814\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.573744156Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"region":"306b53c8-7332-47ac-a835-d7254325a814","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.657012131Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"region":"306b53c8-7332-47ac-a835-d7254325a814","component":"worker"}
{"msg":"region 306b53c8-7332-47ac-a835-d7254325a814 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.657047062Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.657160495Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-306b53c8-7332-47ac-a835-d7254325a814","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.657567365Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-306b53c8-7332-47ac-a835-d7254325a814","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.69949648Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.712114657Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-306b53c8-7332-47ac-a835-d7254325a814","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.715207157Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/306b53c8-7332-47ac-a835-d7254325a814","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.715557466Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.715616168Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.715664119Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.71571573Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.71957117Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.729494277Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:57:26.891472988Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions","method":"POST","req_id":"6fafcece-a493-4f36-acde-af19b89c2d1a","remote_addr":"[fd00:1122:3344:101::a]:62838","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.734568363Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35530"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.734782588Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions/306b53c8-7332-47ac-a835-d7254325a814","method":"GET","req_id":"636f01ad-3d6d-415e-ad57-a2048657f62a","remote_addr":"[fd00:1122:3344:101::a]:35530","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":52,"response_code":200}
{"msg":"path is \"/data/regions/306b53c8-7332-47ac-a835-d7254325a814\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.73869798Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"dataset is oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/306b53c8-7332-47ac-a835-d7254325a814","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.751423179Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"region 306b53c8-7332-47ac-a835-d7254325a814 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.758389419Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.759283253Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.759296643Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions/306b53c8-7332-47ac-a835-d7254325a814","method":"DELETE","req_id":"d60422ac-c50f-417a-b755-e92d34245f3b","remote_addr":"[fd00:1122:3344:101::a]:35530","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20622,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-306b53c8-7332-47ac-a835-d7254325a814 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.760369961Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.762078145Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions/306b53c8-7332-47ac-a835-d7254325a814","method":"GET","req_id":"44db920d-46a6-4634-a8fc-94ff205d490c","remote_addr":"[fd00:1122:3344:101::a]:35530","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":33,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.778378037Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_e34b6fcb-3b3c-487b-ab1a-d89a7d0c9e76/crucible/regions/306b53c8-7332-47ac-a835-d7254325a814\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.786149498Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"region":"306b53c8-7332-47ac-a835-d7254325a814","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.831855831Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions/306b53c8-7332-47ac-a835-d7254325a814","method":"GET","req_id":"4b913589-7d8e-4e39-8f34-9864a54be173","remote_addr":"[fd00:1122:3344:101::a]:35530","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":71,"response_code":200}
{"msg":"region 306b53c8-7332-47ac-a835-d7254325a814 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.868751035Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-12T20:58:17.998230065Z","hostname":"oxz_crucible_4dd1fc4b-308f-49c8-9675-e406d4b18d34","pid":27694,"uri":"/crucible/0/regions/306b53c8-7332-47ac-a835-d7254325a814","method":"GET","req_id":"dc444662-37dd-40f2-a43d-834871517054","remote_addr":"[fd00:1122:3344:101::a]:35530","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":57,"response_code":200}