[ Jul 22 16:15:56 Enabled. ]
[ Jul 22 16:15:56 Rereading configuration. ]
[ Jul 22 16:15:58 Rereading configuration. ]
[ Jul 22 16:16:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 2bfe090eb5318ec8c467157018db2429d4df535b\nCommit timestamp: 2026-06-25T19:33:01.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:03.94770933Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788}
{"msg":"dataset: \"oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:03.947882124Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:03.947893155Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:03.947903655Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:03.967292445Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:04.272836094Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.188293006Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42999"}
{"msg":"region 81033072-bc4b-4606-ab60-ee24e25a1e69 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.217896945Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.22273719Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.225239115Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions","method":"POST","req_id":"e23e73b4-ad72-4ce8-bb3c-a3283950633c","remote_addr":"[fd00:1122:3344:101::a]:42999","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":16252,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.231403604Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.231423414Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"81033072-bc4b-4606-ab60-ee24e25a1e69\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.315451314Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"region":"81033072-bc4b-4606-ab60-ee24e25a1e69","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"81033072-bc4b-4606-ab60-ee24e25a1e69\" \"--data\" \"/data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.317675261Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"region":"81033072-bc4b-4606-ab60-ee24e25a1e69","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.362721114Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions","method":"POST","req_id":"bb8c006d-6add-439a-9203-042d1c5bd043","remote_addr":"[fd00:1122:3344:101::a]:42999","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.478204646Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"region":"81033072-bc4b-4606-ab60-ee24e25a1e69","component":"worker"}
{"msg":"region 81033072-bc4b-4606-ab60-ee24e25a1e69 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.482362683Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.482660831Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.485172566Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.501039366Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.514037801Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.517332746Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.518039645Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.520312743Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.520362895Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.520406016Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.530605639Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.535192698Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:16:41.723977532Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions","method":"POST","req_id":"60dcf0cd-3c9e-40c8-bddb-e6e5616ce3b6","remote_addr":"[fd00:1122:3344:101::a]:42999","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":66,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.843837029Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62277"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.843932442Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","method":"GET","req_id":"823ac3a8-f150-4413-9866-673d63a554a4","remote_addr":"[fd00:1122:3344:101::a]:62277","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":42,"response_code":200}
{"msg":"path is \"/data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.847506704Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"dataset is oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.853654042Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"snapshot is oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69@e776f06b-3e19-435c-8e34-8198239f2f62","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.860434048Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69\", \"e776f06b-3e19-435c-8e34-8198239f2f62\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.860477038Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.871590445Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions/81033072-bc4b-4606-ab60-ee24e25a1e69/snapshots/e776f06b-3e19-435c-8e34-8198239f2f62","method":"GET","req_id":"b7fdd7a2-aef6-4c36-8eb2-8723acf33c22","remote_addr":"[fd00:1122:3344:101::a]:62277","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":24133,"response_code":200}
{"msg":"path is \"/data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.874821839Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"dataset is oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.880704941Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"snapshot is oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69@e776f06b-3e19-435c-8e34-8198239f2f62","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.885778312Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69\", \"e776f06b-3e19-435c-8e34-8198239f2f62\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.885803943Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 81033072-bc4b-4606-ab60-ee24e25a1e69-e776f06b-3e19-435c-8e34-8198239f2f62 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.892530396Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.895527403Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions/81033072-bc4b-4606-ab60-ee24e25a1e69/snapshots/e776f06b-3e19-435c-8e34-8198239f2f62/run","method":"POST","req_id":"1cc55522-fe36-4d2c-9470-74ae8e600eac","remote_addr":"[fd00:1122:3344:101::a]:62277","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20726,"response_code":200}
{"msg":"applying SMF actions for region 81033072-bc4b-4606-ab60-ee24e25a1e69 running snapshot e776f06b-3e19-435c-8e34-8198239f2f62 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.895572905Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.902004111Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.902092943Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.902181455Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.902187935Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.902233077Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.902283378Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.902570685Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.902622347Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.902665108Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.902705289Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.909664239Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-81033072-bc4b-4606-ab60-ee24e25a1e69-e776f06b-3e19-435c-8e34-8198239f2f62","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.912523132Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-81033072-bc4b-4606-ab60-ee24e25a1e69-e776f06b-3e19-435c-8e34-8198239f2f62","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.915470889Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-81033072-bc4b-4606-ab60-ee24e25a1e69-e776f06b-3e19-435c-8e34-8198239f2f62","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.927266633Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69/.zfs/snapshot/e776f06b-3e19-435c-8e34-8198239f2f62","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.927571261Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.927623682Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.927663463Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.927699114Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.927741435Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.932061257Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.936700437Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"region 81033072-bc4b-4606-ab60-ee24e25a1e69 running snapshot e776f06b-3e19-435c-8e34-8198239f2f62 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:50.936725207Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"path is \"/data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:51.15294024Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"dataset is oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:51.159253293Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"snapshot is oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69@e776f06b-3e19-435c-8e34-8198239f2f62","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:51.165072453Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/81033072-bc4b-4606-ab60-ee24e25a1e69\", \"e776f06b-3e19-435c-8e34-8198239f2f62\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:51.165103124Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:51.170257227Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions/81033072-bc4b-4606-ab60-ee24e25a1e69/snapshots/e776f06b-3e19-435c-8e34-8198239f2f62/run","method":"POST","req_id":"4bcbbdd1-482c-4513-9648-49c8e3d01386","remote_addr":"[fd00:1122:3344:101::a]:62277","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":17343,"response_code":200}
{"msg":"region a715008b-cc10-47f5-ac1a-f2a07fd17892 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.704565726Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.704762521Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions","method":"POST","req_id":"4c822e4a-3ae4-4d16-962c-69f35cb140f0","remote_addr":"[fd00:1122:3344:101::a]:62277","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":244,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.704805172Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.710822067Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.710860498Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a715008b-cc10-47f5-ac1a-f2a07fd17892\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.771276788Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"region":"a715008b-cc10-47f5-ac1a-f2a07fd17892","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a715008b-cc10-47f5-ac1a-f2a07fd17892\" \"--data\" \"/data/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.771318499Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"region":"a715008b-cc10-47f5-ac1a-f2a07fd17892","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.843706748Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"region":"a715008b-cc10-47f5-ac1a-f2a07fd17892","component":"worker"}
{"msg":"region a715008b-cc10-47f5-ac1a-f2a07fd17892 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.843738549Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.843895463Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.844497039Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.844591681Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.844681074Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.844704234Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.844742275Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.844787036Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.845179586Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.845223097Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.845266149Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.84530866Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.84933938Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a715008b-cc10-47f5-ac1a-f2a07fd17892","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.849805416Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a715008b-cc10-47f5-ac1a-f2a07fd17892","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.855855482Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.86351658Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a715008b-cc10-47f5-ac1a-f2a07fd17892","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.86740956Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.867750159Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.867811391Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.867853532Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.867892703Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.872335688Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.8790033Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:27:55.883116196Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions","method":"POST","req_id":"c2cff71f-397b-4127-a33e-f2082426ce42","remote_addr":"[fd00:1122:3344:101::a]:62277","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":55,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.345109545Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892","method":"GET","req_id":"ab3b6476-084a-44f6-bc3e-7472838e7a03","remote_addr":"[fd00:1122:3344:101::a]:62277","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":54,"response_code":200}
{"msg":"path is \"/data/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.349248821Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"dataset is oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.358152191Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"region a715008b-cc10-47f5-ac1a-f2a07fd17892 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.365132162Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.365351697Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.367589455Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892","method":"DELETE","req_id":"1202e70a-1480-411d-abdf-5768363afbe1","remote_addr":"[fd00:1122:3344:101::a]:62277","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":18363,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-a715008b-cc10-47f5-ac1a-f2a07fd17892 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.368357355Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.370729286Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892","method":"GET","req_id":"c3581768-1305-4f0f-8fd3-39daab02b24e","remote_addr":"[fd00:1122:3344:101::a]:62277","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":51,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.38210286Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.382197732Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.382312795Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.382324226Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.382379027Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.382433398Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/81033072-bc4b-4606-ab60-ee24e25a1e69","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.382857019Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.382924291Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.382972252Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.383024604Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.391827701Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.393081463Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_1ee80314-a6a7-45dc-9b24-b47d24e373ef/crucible/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892\"","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.399793497Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"region":"a715008b-cc10-47f5-ac1a-f2a07fd17892","component":"worker"}
{"msg":"region a715008b-cc10-47f5-ac1a-f2a07fd17892 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.480351257Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-07-22T16:28:22.503657688Z","hostname":"oxz_crucible_35d1ea17-5b4c-44bd-9d74-c10a9d7eee96","pid":29788,"uri":"/crucible/0/regions/a715008b-cc10-47f5-ac1a-f2a07fd17892","method":"GET","req_id":"2f2185f9-20ad-4b23-b930-d4898b0f506d","remote_addr":"[fd00:1122:3344:101::a]:62277","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":57,"response_code":200}