[ Sep 19 20:46:06 Enabled. ]
[ Sep 19 20:46:06 Rereading configuration. ]
[ Sep 19 20:46:09 Rereading configuration. ]
[ Sep 19 20:46:14 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 779775d5130ff7a4836f52f48b7e64d1479ee104\nCommit timestamp: 2025-08-09T00:27:59.000000000Z  branch: main\nrustc: 1.86.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:46:14.336869721Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044}
{"msg":"dataset: \"oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:46:14.340186979Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:46:14.34021379Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:46:14.34022208Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:46:14.358667415Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:46:14.43101126Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.138487987Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:49627"}
{"msg":"region 07bba53a-55f1-472b-92e1-b4295c66264f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.184038781Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.191005905Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions","method":"POST","req_id":"eba3583e-9b66-4e68-a28e-3ca516b2e357","remote_addr":"[fd00:1122:3344:101::b]:49627","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25403,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.193135805Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.201283908Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.201309958Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"07bba53a-55f1-472b-92e1-b4295c66264f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/07bba53a-55f1-472b-92e1-b4295c66264f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.25738526Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"region":"07bba53a-55f1-472b-92e1-b4295c66264f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"07bba53a-55f1-472b-92e1-b4295c66264f\" \"--data\" \"/data/regions/07bba53a-55f1-472b-92e1-b4295c66264f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.259681564Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"region":"07bba53a-55f1-472b-92e1-b4295c66264f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.509901442Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions","method":"POST","req_id":"77e9e718-6b12-4df3-8ee7-05dde02c2b48","remote_addr":"[fd00:1122:3344:101::b]:49627","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":73,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.743256562Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"region":"07bba53a-55f1-472b-92e1-b4295c66264f","component":"worker"}
{"msg":"region 07bba53a-55f1-472b-92e1-b4295c66264f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.743308023Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.743391995Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.743756853Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.752173022Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.772567322Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.78691727Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.787680308Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.7966967Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.796744731Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.796783252Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.814594652Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.821629478Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:47:07.842766116Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions","method":"POST","req_id":"b123dc0e-41ee-4e35-a4b5-bc2dbe1f1a99","remote_addr":"[fd00:1122:3344:101::b]:49627","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":50,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.622787158Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:62296"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.623391682Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions/07bba53a-55f1-472b-92e1-b4295c66264f","method":"GET","req_id":"61499d68-39a6-44c5-a23d-0fdcf83dc050","remote_addr":"[fd00:1122:3344:101::b]:62296","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":52,"response_code":200}
{"msg":"path is \"/data/regions/07bba53a-55f1-472b-92e1-b4295c66264f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.628713507Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"dataset is oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.637173827Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"snapshot is oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f@bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.645040242Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f\", \"bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.645074813Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.657550317Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions/07bba53a-55f1-472b-92e1-b4295c66264f/snapshots/bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a","method":"GET","req_id":"97c5df5a-2562-48e5-9988-eddc723c8c84","remote_addr":"[fd00:1122:3344:101::b]:62296","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":31514,"response_code":200}
{"msg":"path is \"/data/regions/07bba53a-55f1-472b-92e1-b4295c66264f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.661235154Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"dataset is oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.668803642Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"snapshot is oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f@bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.676169976Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f\", \"bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.676204437Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 07bba53a-55f1-472b-92e1-b4295c66264f-bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.685473585Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"applying SMF actions for region 07bba53a-55f1-472b-92e1-b4295c66264f running snapshot bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.68609718Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.68611213Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions/07bba53a-55f1-472b-92e1-b4295c66264f/snapshots/bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a/run","method":"POST","req_id":"473f420f-3766-4113-a3c0-c559349caa3c","remote_addr":"[fd00:1122:3344:101::b]:62296","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24912,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.706553452Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.706636164Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.706710546Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.706718126Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.706755557Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.706806518Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.70691653Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.706993162Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.707047153Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.707101225Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.718229057Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-07bba53a-55f1-472b-92e1-b4295c66264f-bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.718750659Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-07bba53a-55f1-472b-92e1-b4295c66264f-bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.729213326Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-07bba53a-55f1-472b-92e1-b4295c66264f-bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.748719976Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/07bba53a-55f1-472b-92e1-b4295c66264f/.zfs/snapshot/bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.749026423Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.749090024Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.749132276Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.749177466Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.749235758Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.756740985Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.764632931Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"region 07bba53a-55f1-472b-92e1-b4295c66264f running snapshot bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:50.764650291Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"path is \"/data/regions/07bba53a-55f1-472b-92e1-b4295c66264f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:51.028824487Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"dataset is oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:51.038687139Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"snapshot is oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f@bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:51.047285982Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/07bba53a-55f1-472b-92e1-b4295c66264f\", \"bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:51.047340713Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:51.055591898Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions/07bba53a-55f1-472b-92e1-b4295c66264f/snapshots/bddf0dcb-bd19-4f2c-8c1b-3b9508c0755a/run","method":"POST","req_id":"416e2750-440d-4454-924c-0a8584b5dda5","remote_addr":"[fd00:1122:3344:101::b]:62296","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26796,"response_code":200}
{"msg":"region 3b4ec30b-1c27-4606-aec0-d32b6645b226 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.48077793Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.480966675Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions","method":"POST","req_id":"6f1a7b01-5709-445c-9fff-a1ba2051aebf","remote_addr":"[fd00:1122:3344:101::b]:62296","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":246,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.480995346Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.48925384Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.489298121Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3b4ec30b-1c27-4606-aec0-d32b6645b226\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.544722577Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"region":"3b4ec30b-1c27-4606-aec0-d32b6645b226","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3b4ec30b-1c27-4606-aec0-d32b6645b226\" \"--data\" \"/data/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.546599952Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"region":"3b4ec30b-1c27-4606-aec0-d32b6645b226","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.668113936Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"region":"3b4ec30b-1c27-4606-aec0-d32b6645b226","component":"worker"}
{"msg":"region 3b4ec30b-1c27-4606-aec0-d32b6645b226 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.668166467Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.66831809Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.668963876Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.669044038Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.669115669Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.66912882Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.66916141Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.669210722Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.6695772Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.669639472Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.669685193Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.669728194Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.676958624Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3b4ec30b-1c27-4606-aec0-d32b6645b226","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.677462606Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3b4ec30b-1c27-4606-aec0-d32b6645b226","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.681496891Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.694676362Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3b4ec30b-1c27-4606-aec0-d32b6645b226","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.70094892Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.701314088Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.701418271Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.701503423Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.701576574Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.707079924Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.715440031Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:57:56.817485375Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions","method":"POST","req_id":"a81f90e3-0a2a-44ff-8df9-74fb3dac07ee","remote_addr":"[fd00:1122:3344:101::b]:62296","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":63,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.320180022Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226","method":"GET","req_id":"cc132337-1185-4562-af6c-b51459833f46","remote_addr":"[fd00:1122:3344:101::b]:62296","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":48,"response_code":200}
{"msg":"path is \"/data/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.322736222Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"dataset is oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.336047686Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"region 3b4ec30b-1c27-4606-aec0-d32b6645b226 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.347113677Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.347296391Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.347330062Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226","method":"DELETE","req_id":"8093a969-c0a2-47fa-9eb1-257c59439bac","remote_addr":"[fd00:1122:3344:101::b]:62296","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24661,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-3b4ec30b-1c27-4606-aec0-d32b6645b226 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.348319685Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.350718112Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226","method":"GET","req_id":"04f38f39-bcbd-4dab-b55e-19d1d108a0e6","remote_addr":"[fd00:1122:3344:101::b]:62296","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":34,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.357449431Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.357524772Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.357600304Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.357617455Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.357642705Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.357697277Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/07bba53a-55f1-472b-92e1-b4295c66264f","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.358050655Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.358119056Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.358180178Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.358237639Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.362827217Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.364115398Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ed1df88a-1a8d-46df-b619-be4f0abc53e1/crucible/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.374349349Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"region":"3b4ec30b-1c27-4606-aec0-d32b6645b226","component":"worker"}
{"msg":"region 3b4ec30b-1c27-4606-aec0-d32b6645b226 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.430550354Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-19T20:58:22.441212545Z","hostname":"oxz_crucible_9f244d9b-a92f-4cf3-be19-c240f6279556","pid":26044,"uri":"/crucible/0/regions/3b4ec30b-1c27-4606-aec0-d32b6645b226","method":"GET","req_id":"c39cfeae-c323-430d-95ed-e1ac84130fa9","remote_addr":"[fd00:1122:3344:101::b]:62296","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":64,"response_code":200}