[ Jun  2 18:01:03 Enabled. ]
[ Jun  2 18:01:03 Rereading configuration. ]
[ Jun  2 18:01:05 Rereading configuration. ]
[ Jun  2 18:01:08 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_92229be4-2175-4640-9226-225651c4f571/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: bd9a0e2abe6b6b89aec8c85f4ee57474144ed150\nCommit timestamp: 2026-05-14T15:48:00.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:08.797415781Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256}
{"msg":"dataset: \"oxp_92229be4-2175-4640-9226-225651c4f571/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:08.797640957Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:08.797657587Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:08.797664987Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:08.816701001Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:08.892742986Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.244120502Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:65458"}
{"msg":"region 26e23ba7-b204-4124-96d6-67d482d54718 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.282678882Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.287252046Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.287325458Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions","method":"POST","req_id":"3bfc88cd-7a93-43d1-a6eb-bb192bb7c3f1","remote_addr":"[fd00:1122:3344:101::a]:65458","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":13630,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.294082246Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.294106047Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"26e23ba7-b204-4124-96d6-67d482d54718\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/26e23ba7-b204-4124-96d6-67d482d54718\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.371009543Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"region":"26e23ba7-b204-4124-96d6-67d482d54718","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"26e23ba7-b204-4124-96d6-67d482d54718\" \"--data\" \"/data/regions/26e23ba7-b204-4124-96d6-67d482d54718\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.373050003Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"region":"26e23ba7-b204-4124-96d6-67d482d54718","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.503876532Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"region":"26e23ba7-b204-4124-96d6-67d482d54718","component":"worker"}
{"msg":"region 26e23ba7-b204-4124-96d6-67d482d54718 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.503956174Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.50419789Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.506651061Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.51182992Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.511899432Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions","method":"POST","req_id":"5a0473e4-1b56-4430-a547-f9a50b2bb34e","remote_addr":"[fd00:1122:3344:101::a]:65458","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":65,"response_code":200}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.525126381Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.52709723Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.527853679Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.527919081Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.527946501Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.528013903Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.538079094Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:01:40.541063128Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.607028219Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35719"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.607151612Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions/26e23ba7-b204-4124-96d6-67d482d54718","method":"GET","req_id":"5f08a38a-0813-4c24-81ea-e27a3bc9e01b","remote_addr":"[fd00:1122:3344:101::a]:35719","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":47,"response_code":200}
{"msg":"path is \"/data/regions/26e23ba7-b204-4124-96d6-67d482d54718\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.610786233Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"dataset is oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.61749858Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"snapshot is oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718@6a0d8d79-ab18-4a70-8452-38e82654f96e","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.623087729Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718\", \"6a0d8d79-ab18-4a70-8452-38e82654f96e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.62309967Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.631709484Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions/26e23ba7-b204-4124-96d6-67d482d54718/snapshots/6a0d8d79-ab18-4a70-8452-38e82654f96e","method":"GET","req_id":"959cd033-2e2f-423f-a603-5b0bf499a29f","remote_addr":"[fd00:1122:3344:101::a]:35719","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":20944,"response_code":200}
{"msg":"path is \"/data/regions/26e23ba7-b204-4124-96d6-67d482d54718\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.635447937Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"dataset is oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.640836171Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"snapshot is oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718@6a0d8d79-ab18-4a70-8452-38e82654f96e","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.647312482Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718\", \"6a0d8d79-ab18-4a70-8452-38e82654f96e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.647326493Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 26e23ba7-b204-4124-96d6-67d482d54718-6a0d8d79-ab18-4a70-8452-38e82654f96e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.655043815Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"applying SMF actions for region 26e23ba7-b204-4124-96d6-67d482d54718 running snapshot 6a0d8d79-ab18-4a70-8452-38e82654f96e (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.658340657Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.658356438Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions/26e23ba7-b204-4124-96d6-67d482d54718/snapshots/6a0d8d79-ab18-4a70-8452-38e82654f96e/run","method":"POST","req_id":"7fc778f3-d21b-48aa-976a-75be60b76833","remote_addr":"[fd00:1122:3344:101::a]:35719","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":22945,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.667192208Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.6672661Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.667338332Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.667344492Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.667381222Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.667422654Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.667533096Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.667583247Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.667626239Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.66766713Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.673018543Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-26e23ba7-b204-4124-96d6-67d482d54718-6a0d8d79-ab18-4a70-8452-38e82654f96e","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.675921955Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-26e23ba7-b204-4124-96d6-67d482d54718-6a0d8d79-ab18-4a70-8452-38e82654f96e","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.681929865Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-26e23ba7-b204-4124-96d6-67d482d54718-6a0d8d79-ab18-4a70-8452-38e82654f96e","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.689291328Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/26e23ba7-b204-4124-96d6-67d482d54718/.zfs/snapshot/6a0d8d79-ab18-4a70-8452-38e82654f96e","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.689617567Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.689647437Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.689692018Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.689728659Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.68977475Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.691791401Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.694707443Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"region 26e23ba7-b204-4124-96d6-67d482d54718 running snapshot 6a0d8d79-ab18-4a70-8452-38e82654f96e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:46.694721454Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"path is \"/data/regions/26e23ba7-b204-4124-96d6-67d482d54718\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:47.015728951Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"dataset is oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:47.022018707Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"snapshot is oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718@6a0d8d79-ab18-4a70-8452-38e82654f96e","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:47.029195276Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/26e23ba7-b204-4124-96d6-67d482d54718\", \"6a0d8d79-ab18-4a70-8452-38e82654f96e\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:47.029223687Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:47.034093038Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions/26e23ba7-b204-4124-96d6-67d482d54718/snapshots/6a0d8d79-ab18-4a70-8452-38e82654f96e/run","method":"POST","req_id":"5d7f1650-8dab-4c15-9f5b-e696489f1b3c","remote_addr":"[fd00:1122:3344:101::a]:35719","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":18406,"response_code":200}
{"msg":"region 0692fa66-2028-4980-9ccc-8c887484377a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.28570257Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.286769917Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions","method":"POST","req_id":"9c2e2bf7-3346-4070-b605-9b6064559c04","remote_addr":"[fd00:1122:3344:101::a]:35719","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":312,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.286813958Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/0692fa66-2028-4980-9ccc-8c887484377a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.292449689Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/0692fa66-2028-4980-9ccc-8c887484377a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.292465119Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0692fa66-2028-4980-9ccc-8c887484377a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0692fa66-2028-4980-9ccc-8c887484377a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.33785299Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"region":"0692fa66-2028-4980-9ccc-8c887484377a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0692fa66-2028-4980-9ccc-8c887484377a\" \"--data\" \"/data/regions/0692fa66-2028-4980-9ccc-8c887484377a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.337896551Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"region":"0692fa66-2028-4980-9ccc-8c887484377a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.382330738Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"region":"0692fa66-2028-4980-9ccc-8c887484377a","component":"worker"}
{"msg":"region 0692fa66-2028-4980-9ccc-8c887484377a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.382356328Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.382494232Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0692fa66-2028-4980-9ccc-8c887484377a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.382873311Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0692fa66-2028-4980-9ccc-8c887484377a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.386108252Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.389392034Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0692fa66-2028-4980-9ccc-8c887484377a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.390942662Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0692fa66-2028-4980-9ccc-8c887484377a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.391216409Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.391268Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.391306581Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.391341682Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.392897631Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.395815594Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.395913006Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.395992358Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.396000198Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.396038459Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.39608231Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.398515261Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.398574322Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.398639514Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.398682395Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.400427479Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.402423228Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:12:51.592687897Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions","method":"POST","req_id":"8f499ce5-00ae-4ea0-a356-60b556b8a735","remote_addr":"[fd00:1122:3344:101::a]:35719","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":92,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.879180322Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions/0692fa66-2028-4980-9ccc-8c887484377a","method":"GET","req_id":"4fa01c0f-322c-49c6-8bbd-5656be4960d1","remote_addr":"[fd00:1122:3344:101::a]:35719","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":68,"response_code":200}
{"msg":"path is \"/data/regions/0692fa66-2028-4980-9ccc-8c887484377a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.882565706Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"dataset is oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/0692fa66-2028-4980-9ccc-8c887484377a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.894403741Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"region 0692fa66-2028-4980-9ccc-8c887484377a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.901434696Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.90159316Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.901612231Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions/0692fa66-2028-4980-9ccc-8c887484377a","method":"DELETE","req_id":"87361292-72ff-4f04-b99a-2163b11feb5e","remote_addr":"[fd00:1122:3344:101::a]:35719","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19077,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-0692fa66-2028-4980-9ccc-8c887484377a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.902685007Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.904069872Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions/0692fa66-2028-4980-9ccc-8c887484377a","method":"GET","req_id":"e7b5561c-ed46-469c-90db-a43f46721de9","remote_addr":"[fd00:1122:3344:101::a]:35719","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":34,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.905485827Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.90558467Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.905664902Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.905687992Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.905701773Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.905748884Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/26e23ba7-b204-4124-96d6-67d482d54718","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.906086502Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.906153784Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.906210825Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.906253186Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.908871822Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.910185354Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_92229be4-2175-4640-9226-225651c4f571/crucible/regions/0692fa66-2028-4980-9ccc-8c887484377a\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.918224255Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"region":"0692fa66-2028-4980-9ccc-8c887484377a","component":"worker"}
{"msg":"region 0692fa66-2028-4980-9ccc-8c887484377a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:15.969463641Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-02T18:13:16.02200362Z","hostname":"oxz_crucible_fac6c762-146d-483e-8054-0588e55d1075","pid":24256,"uri":"/crucible/0/regions/0692fa66-2028-4980-9ccc-8c887484377a","method":"GET","req_id":"da7f2f5a-b130-43f9-9da8-cb5c9dc9aee3","remote_addr":"[fd00:1122:3344:101::a]:35719","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":51,"response_code":200}