[ Jun 16 22:30:34 Enabled. ]
[ Jun 16 22:30:34 Rereading configuration. ]
[ Jun 16 22:30:39 Rereading configuration. ]
[ Jun 16 22:30:45 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: bd9a0e2abe6b6b89aec8c85f4ee57474144ed150\nCommit timestamp: 2026-05-14T15:48:00.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:30:45.503644781Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299}
{"msg":"dataset: \"oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:30:45.503896504Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:30:45.503973824Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:30:45.503979944Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:30:45.51980146Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:30:45.691263585Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.73469496Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51736"}
{"msg":"region cc773809-e453-484a-9502-dc6a00034744 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.768811375Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.772872635Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.772902275Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions","method":"POST","req_id":"d615827d-540b-4a7f-9d78-89b0cbc83983","remote_addr":"[fd00:1122:3344:101::a]:51736","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":12655,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.781006365Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.781027115Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"cc773809-e453-484a-9502-dc6a00034744\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/cc773809-e453-484a-9502-dc6a00034744\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.844674411Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"region":"cc773809-e453-484a-9502-dc6a00034744","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"cc773809-e453-484a-9502-dc6a00034744\" \"--data\" \"/data/regions/cc773809-e453-484a-9502-dc6a00034744\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.84663151Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"region":"cc773809-e453-484a-9502-dc6a00034744","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.961121965Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions","method":"POST","req_id":"7213dd79-ab59-4455-9a8c-6399ec0fc323","remote_addr":"[fd00:1122:3344:101::a]:51736","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.998769505Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"region":"cc773809-e453-484a-9502-dc6a00034744","component":"worker"}
{"msg":"region cc773809-e453-484a-9502-dc6a00034744 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.998803335Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:45.998915936Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.001265369Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.005807224Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.019170865Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.023119664Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.02383336Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.023891271Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.023935141Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.023973552Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.03498178Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.039442654Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:31:46.498874129Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions","method":"POST","req_id":"bda4b977-30a8-4f2b-8b71-a5c41ab764c2","remote_addr":"[fd00:1122:3344:101::a]:51736","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":74,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.482819218Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56594"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.482910349Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions/cc773809-e453-484a-9502-dc6a00034744","method":"GET","req_id":"e872da4a-c2b2-489b-a8b6-9abab5ccd2b1","remote_addr":"[fd00:1122:3344:101::a]:56594","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":43,"response_code":200}
{"msg":"path is \"/data/regions/cc773809-e453-484a-9502-dc6a00034744\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.485289112Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"dataset is oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.490790536Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"snapshot is oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744@a415edd3-7801-4052-9db1-0edde976693a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.496926316Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744\", \"a415edd3-7801-4052-9db1-0edde976693a\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.496957907Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.509332598Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions/cc773809-e453-484a-9502-dc6a00034744/snapshots/a415edd3-7801-4052-9db1-0edde976693a","method":"GET","req_id":"69701d71-ce0e-4c6b-b001-a2e17d0753d3","remote_addr":"[fd00:1122:3344:101::a]:56594","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":24050,"response_code":200}
{"msg":"path is \"/data/regions/cc773809-e453-484a-9502-dc6a00034744\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.512704932Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"dataset is oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.517844832Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"snapshot is oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744@a415edd3-7801-4052-9db1-0edde976693a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.523584888Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744\", \"a415edd3-7801-4052-9db1-0edde976693a\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.523602919Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot cc773809-e453-484a-9502-dc6a00034744-a415edd3-7801-4052-9db1-0edde976693a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.533606467Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"applying SMF actions for region cc773809-e453-484a-9502-dc6a00034744 running snapshot a415edd3-7801-4052-9db1-0edde976693a (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.536906159Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.536917879Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions/cc773809-e453-484a-9502-dc6a00034744/snapshots/a415edd3-7801-4052-9db1-0edde976693a/run","method":"POST","req_id":"ed4dfbb9-73cf-4987-b095-feb8b71f0b92","remote_addr":"[fd00:1122:3344:101::a]:56594","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":24228,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.545390173Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.545468254Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.545542224Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.545549874Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.545590045Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.545632085Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.545721746Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.545769476Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.545812047Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.545852877Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.555112968Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-cc773809-e453-484a-9502-dc6a00034744-a415edd3-7801-4052-9db1-0edde976693a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.557807435Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-cc773809-e453-484a-9502-dc6a00034744-a415edd3-7801-4052-9db1-0edde976693a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.563778573Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-cc773809-e453-484a-9502-dc6a00034744-a415edd3-7801-4052-9db1-0edde976693a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.581985122Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cc773809-e453-484a-9502-dc6a00034744/.zfs/snapshot/a415edd3-7801-4052-9db1-0edde976693a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.582373916Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.582436837Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.582482917Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.582530168Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.582582968Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.587295945Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.594571936Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"region cc773809-e453-484a-9502-dc6a00034744 running snapshot a415edd3-7801-4052-9db1-0edde976693a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.594590166Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"path is \"/data/regions/cc773809-e453-484a-9502-dc6a00034744\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.763375635Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"dataset is oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.770881499Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"snapshot is oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744@a415edd3-7801-4052-9db1-0edde976693a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.77813858Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/cc773809-e453-484a-9502-dc6a00034744\", \"a415edd3-7801-4052-9db1-0edde976693a\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.778185961Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:40.784551983Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions/cc773809-e453-484a-9502-dc6a00034744/snapshots/a415edd3-7801-4052-9db1-0edde976693a/run","method":"POST","req_id":"5437e766-c681-4931-8dc0-3d74419cf7cd","remote_addr":"[fd00:1122:3344:101::a]:56594","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21220,"response_code":200}
{"msg":"region 03bc5838-0d42-46bf-b78f-f1a394094c02 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.198595495Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.198793288Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.198813978Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions","method":"POST","req_id":"a3a3e046-435f-4608-b0f4-8b896b24fe5e","remote_addr":"[fd00:1122:3344:101::a]:56594","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":244,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/03bc5838-0d42-46bf-b78f-f1a394094c02","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.204926698Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/03bc5838-0d42-46bf-b78f-f1a394094c02","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.204950238Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"03bc5838-0d42-46bf-b78f-f1a394094c02\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/03bc5838-0d42-46bf-b78f-f1a394094c02\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.262197971Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"region":"03bc5838-0d42-46bf-b78f-f1a394094c02","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"03bc5838-0d42-46bf-b78f-f1a394094c02\" \"--data\" \"/data/regions/03bc5838-0d42-46bf-b78f-f1a394094c02\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.262240821Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"region":"03bc5838-0d42-46bf-b78f-f1a394094c02","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.373208651Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"region":"03bc5838-0d42-46bf-b78f-f1a394094c02","component":"worker"}
{"msg":"region 03bc5838-0d42-46bf-b78f-f1a394094c02 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.373240752Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.373412164Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-03bc5838-0d42-46bf-b78f-f1a394094c02","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.373782107Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-03bc5838-0d42-46bf-b78f-f1a394094c02","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.382682855Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.395973145Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-03bc5838-0d42-46bf-b78f-f1a394094c02","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.402021565Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/03bc5838-0d42-46bf-b78f-f1a394094c02","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.402329628Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.402384408Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.402409338Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.402449109Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.40866963Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.415128423Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.415246925Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.415339906Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.415354556Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.415394676Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.415448787Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.423702148Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.423760358Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.423809429Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.423860589Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.43106144Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.432840798Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:42:46.523173365Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions","method":"POST","req_id":"d8f3e24b-4b72-462d-8b8e-48c5cf660ad3","remote_addr":"[fd00:1122:3344:101::a]:56594","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":57,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:11.977810074Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions/03bc5838-0d42-46bf-b78f-f1a394094c02","method":"GET","req_id":"d95b466f-327b-4d9b-b4b8-39c49bb392df","remote_addr":"[fd00:1122:3344:101::a]:56594","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":51,"response_code":200}
{"msg":"path is \"/data/regions/03bc5838-0d42-46bf-b78f-f1a394094c02\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:11.981125656Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"dataset is oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/03bc5838-0d42-46bf-b78f-f1a394094c02","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:11.991265586Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"region 03bc5838-0d42-46bf-b78f-f1a394094c02 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:11.998684729Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:11.998876801Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:11.998897411Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions/03bc5838-0d42-46bf-b78f-f1a394094c02","method":"DELETE","req_id":"a5275178-4fc9-4faa-8449-9f4ae4210ef8","remote_addr":"[fd00:1122:3344:101::a]:56594","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":17792,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-03bc5838-0d42-46bf-b78f-f1a394094c02 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:11.999430686Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.00289572Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions/03bc5838-0d42-46bf-b78f-f1a394094c02","method":"GET","req_id":"d0e28a66-1c6e-447c-88e5-7579f4afe33f","remote_addr":"[fd00:1122:3344:101::a]:56594","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":45,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.005251243Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.005327684Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.005413985Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.005426135Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.005463925Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.005515376Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/cc773809-e453-484a-9502-dc6a00034744","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.005844239Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.00590759Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.00596545Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.006023211Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.013440424Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.014631255Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f7d7207a-da49-409b-9529-fddaa9473589/crucible/regions/03bc5838-0d42-46bf-b78f-f1a394094c02\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.024096268Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"region":"03bc5838-0d42-46bf-b78f-f1a394094c02","component":"worker"}
{"msg":"region 03bc5838-0d42-46bf-b78f-f1a394094c02 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.083828415Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-16T22:43:12.121233282Z","hostname":"oxz_crucible_1197555a-8ad9-40af-ba7b-68d6f1e81418","pid":28299,"uri":"/crucible/0/regions/03bc5838-0d42-46bf-b78f-f1a394094c02","method":"GET","req_id":"c44eafe5-b529-48aa-8cde-eabd7ca98616","remote_addr":"[fd00:1122:3344:101::a]:56594","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":56,"response_code":200}