[ Mar 26 15:17:39 Enabled. ]
[ Mar 26 15:17:39 Rereading configuration. ]
[ Mar 26 15:17:42 Rereading configuration. ]
[ Mar 26 15:17:48 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:17:48.942685542Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173}
{"msg":"dataset: \"oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:17:48.943428262Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:17:48.943438872Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:17:48.943444292Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:17:48.955784432Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:17:49.066619279Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:41.848790464Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:52525"}
{"msg":"region 4368dcc5-672a-4f5c-b8a1-e348606c9118 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:41.892206197Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:41.894980743Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:41.895002573Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions","method":"POST","req_id":"7e330375-44b7-471e-bea3-60f1a5d39939","remote_addr":"[fd00:1122:3344:101::c]:52525","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":18681,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:41.903897589Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:41.903914599Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4368dcc5-672a-4f5c-b8a1-e348606c9118\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:41.968913612Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"region":"4368dcc5-672a-4f5c-b8a1-e348606c9118","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4368dcc5-672a-4f5c-b8a1-e348606c9118\" \"--data\" \"/data/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:41.971296343Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"region":"4368dcc5-672a-4f5c-b8a1-e348606c9118","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.194652579Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions","method":"POST","req_id":"e7bdc15e-5bad-4e66-9e53-cbb1e727bd97","remote_addr":"[fd00:1122:3344:101::c]:52525","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.322422115Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"region":"4368dcc5-672a-4f5c-b8a1-e348606c9118","component":"worker"}
{"msg":"region 4368dcc5-672a-4f5c-b8a1-e348606c9118 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.322495576Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.322683718Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4368dcc5-672a-4f5c-b8a1-e348606c9118","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.323007522Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4368dcc5-672a-4f5c-b8a1-e348606c9118","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.330424308Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.345135539Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4368dcc5-672a-4f5c-b8a1-e348606c9118","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.348815487Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.349563517Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.349632068Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.349678168Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.349714149Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.354031535Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.359115561Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:18:42.733588786Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions","method":"POST","req_id":"6c65a0b2-6985-4d7b-9651-785cdd6f6e11","remote_addr":"[fd00:1122:3344:101::c]:52525","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.294216926Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:65224"}
{"msg":"region 164f3c04-fc99-4697-a693-c2290cefcb49 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.294905035Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.295162348Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.295183278Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions","method":"POST","req_id":"77a8a3e1-7ea1-4677-8b7a-77a27f89a05f","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":850,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/164f3c04-fc99-4697-a693-c2290cefcb49","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.305006466Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/164f3c04-fc99-4697-a693-c2290cefcb49","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.305039036Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"164f3c04-fc99-4697-a693-c2290cefcb49\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/164f3c04-fc99-4697-a693-c2290cefcb49\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.363028528Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"region":"164f3c04-fc99-4697-a693-c2290cefcb49","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"164f3c04-fc99-4697-a693-c2290cefcb49\" \"--data\" \"/data/regions/164f3c04-fc99-4697-a693-c2290cefcb49\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.363061459Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"region":"164f3c04-fc99-4697-a693-c2290cefcb49","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.439686332Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"region":"164f3c04-fc99-4697-a693-c2290cefcb49","component":"worker"}
{"msg":"region 164f3c04-fc99-4697-a693-c2290cefcb49 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.439714712Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.439812054Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-164f3c04-fc99-4697-a693-c2290cefcb49","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.440131198Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-164f3c04-fc99-4697-a693-c2290cefcb49","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.45104111Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.473885106Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-164f3c04-fc99-4697-a693-c2290cefcb49","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.476919675Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/164f3c04-fc99-4697-a693-c2290cefcb49","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.477189418Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.477240199Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.47728417Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.4773192Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.481390893Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.495996212Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:54.627472087Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions","method":"POST","req_id":"684c1929-eb35-4ad6-8040-8d63ff110cab","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":85,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.714944108Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118","method":"GET","req_id":"379651c7-10d1-4530-9e1a-b81e04538140","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":68,"response_code":200}
{"msg":"path is \"/data/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.718608116Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"dataset is oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.725280932Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"snapshot is oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118@41a764fc-3d99-48fd-a6cb-ced9d1fe2758","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.732334144Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118\", \"41a764fc-3d99-48fd-a6cb-ced9d1fe2758\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.732356404Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.741804576Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118/snapshots/41a764fc-3d99-48fd-a6cb-ced9d1fe2758","method":"GET","req_id":"4ada6ae6-c10f-41c4-aaf7-2b057470f78f","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":23234,"response_code":200}
{"msg":"path is \"/data/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.745031908Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"dataset is oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.751245879Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"snapshot is oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118@41a764fc-3d99-48fd-a6cb-ced9d1fe2758","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.757978536Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118\", \"41a764fc-3d99-48fd-a6cb-ced9d1fe2758\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.757995146Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 4368dcc5-672a-4f5c-b8a1-e348606c9118-41a764fc-3d99-48fd-a6cb-ced9d1fe2758 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.765382322Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.765511874Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118/snapshots/41a764fc-3d99-48fd-a6cb-ced9d1fe2758/run","method":"POST","req_id":"4ba9b55d-d35e-4475-96a4-978f08cee10e","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20504,"response_code":200}
{"msg":"applying SMF actions for region 4368dcc5-672a-4f5c-b8a1-e348606c9118 running snapshot 41a764fc-3d99-48fd-a6cb-ced9d1fe2758 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.765527634Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-4368dcc5-672a-4f5c-b8a1-e348606c9118-41a764fc-3d99-48fd-a6cb-ced9d1fe2758","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.769781059Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-4368dcc5-672a-4f5c-b8a1-e348606c9118-41a764fc-3d99-48fd-a6cb-ced9d1fe2758","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.782740437Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-4368dcc5-672a-4f5c-b8a1-e348606c9118-41a764fc-3d99-48fd-a6cb-ced9d1fe2758","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.793049231Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118/.zfs/snapshot/41a764fc-3d99-48fd-a6cb-ced9d1fe2758","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.793343795Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.793386585Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.793425976Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.793460466Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.793501187Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.797275386Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.801233077Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"region 4368dcc5-672a-4f5c-b8a1-e348606c9118 running snapshot 41a764fc-3d99-48fd-a6cb-ced9d1fe2758 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:55.801255877Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"path is \"/data/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:56.113201962Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"dataset is oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:56.120034401Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"snapshot is oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118@41a764fc-3d99-48fd-a6cb-ced9d1fe2758","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:56.126458944Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118\", \"41a764fc-3d99-48fd-a6cb-ced9d1fe2758\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:56.126482044Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:28:56.132612494Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions/4368dcc5-672a-4f5c-b8a1-e348606c9118/snapshots/41a764fc-3d99-48fd-a6cb-ced9d1fe2758/run","method":"POST","req_id":"5b54d6f0-1dbe-4ee2-9899-3e469c68d7f0","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19459,"response_code":200}
{"msg":"region 97cede2e-a40f-468e-a3f2-59a95dee89c8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.619031809Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.619201572Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions","method":"POST","req_id":"0434aa5a-2a6a-4c0b-acbc-fdac11a22083","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":214,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.619225162Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.625502173Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.625524523Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"97cede2e-a40f-468e-a3f2-59a95dee89c8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.684847913Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"region":"97cede2e-a40f-468e-a3f2-59a95dee89c8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"97cede2e-a40f-468e-a3f2-59a95dee89c8\" \"--data\" \"/data/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.684888123Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"region":"97cede2e-a40f-468e-a3f2-59a95dee89c8","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.756634238Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"region":"97cede2e-a40f-468e-a3f2-59a95dee89c8","component":"worker"}
{"msg":"region 97cede2e-a40f-468e-a3f2-59a95dee89c8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.756660357Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.756831356Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-97cede2e-a40f-468e-a3f2-59a95dee89c8","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.758481618Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-97cede2e-a40f-468e-a3f2-59a95dee89c8","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.761809261Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.772278306Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-97cede2e-a40f-468e-a3f2-59a95dee89c8","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.775773572Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.776067426Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.776115146Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.776155537Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.776195117Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.779730963Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.785122223Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:01.811618567Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions","method":"POST","req_id":"f5ad5b9a-c739-4c3d-8144-c680d4d122f0","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":50,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.49645393Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8","method":"GET","req_id":"67e88371-bed8-454d-931a-b082d93e5a2f","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":68,"response_code":200}
{"msg":"path is \"/data/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.498608668Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"dataset is oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.507005717Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"region 97cede2e-a40f-468e-a3f2-59a95dee89c8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.515006921Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.515189153Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8","method":"DELETE","req_id":"4afe4d48-0027-4e04-9887-f950a6144971","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":16613,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.515234814Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-97cede2e-a40f-468e-a3f2-59a95dee89c8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.516233737Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.519347367Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8","method":"GET","req_id":"153c269e-f305-4eb0-bbbc-d61fe343a31a","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":32,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.526577811Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8541defb-f7cc-49eb-9f05-247131d7cb9c/crucible/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8\"","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.533740994Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"region":"97cede2e-a40f-468e-a3f2-59a95dee89c8","component":"worker"}
{"msg":"region 97cede2e-a40f-468e-a3f2-59a95dee89c8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.592769409Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-03-26T15:29:27.61827115Z","hostname":"oxz_crucible_89aa4a6d-b344-480c-b312-13d45bc0a4cd","pid":24173,"uri":"/crucible/0/regions/97cede2e-a40f-468e-a3f2-59a95dee89c8","method":"GET","req_id":"44d622a6-4464-4b9c-8555-8c16d429bbb6","remote_addr":"[fd00:1122:3344:101::c]:65224","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":200}