[ Jun 25 23:15:16 Enabled. ]
[ Jun 25 23:15:16 Rereading configuration. ]
[ Jun 25 23:15:19 Rereading configuration. ]
[ Jun 25 23:15:22 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 2bfe090eb5318ec8c467157018db2429d4df535b\nCommit timestamp: 2026-06-25T19:33:01.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:15:23.027790047Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481}
{"msg":"dataset: \"oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:15:23.028025268Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:15:23.028042918Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:15:23.028050018Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:15:23.050944527Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:15:23.128769293Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.302558999Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46127"}
{"msg":"region 92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.330514306Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.335026831Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.337365494Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions","method":"POST","req_id":"e7e34b92-7a35-4835-a60c-401797caba65","remote_addr":"[fd00:1122:3344:101::a]:46127","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":15191,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.343516444Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.343534078Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.394525075Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"region":"92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51\" \"--data\" \"/data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.396582036Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"region":"92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.550950621Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"region":"92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","component":"worker"}
{"msg":"region 92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.552892422Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.553028912Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.555230365Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.564376893Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.576799086Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.581872454Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.582601848Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.584578999Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.584624069Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.58466755Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.594841027Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.59897511Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:16:25.656775364Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions","method":"POST","req_id":"81ad6344-0b98-4735-a786-ed7bc275ecfe","remote_addr":"[fd00:1122:3344:101::a]:46127","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":71,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.802757223Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:38840"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.802917694Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","method":"GET","req_id":"f404daca-a6ef-40f3-8388-99e5037a0377","remote_addr":"[fd00:1122:3344:101::a]:38840","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":80,"response_code":200}
{"msg":"path is \"/data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.80768022Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"dataset is oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.817350144Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"snapshot is oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51@be250217-78d8-4efa-928c-110459283db3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.827538812Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51\", \"be250217-78d8-4efa-928c-110459283db3\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.827573052Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.843599642Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51/snapshots/be250217-78d8-4efa-928c-110459283db3","method":"GET","req_id":"f63399bf-fab0-4cc2-8f27-0c09e226eb60","remote_addr":"[fd00:1122:3344:101::a]:38840","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":35962,"response_code":200}
{"msg":"path is \"/data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.847797625Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"dataset is oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.858132363Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"snapshot is oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51@be250217-78d8-4efa-928c-110459283db3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.869125625Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51\", \"be250217-78d8-4efa-928c-110459283db3\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.869170745Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51-be250217-78d8-4efa-928c-110459283db3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.879955605Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"applying SMF actions for region 92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51 running snapshot be250217-78d8-4efa-928c-110459283db3 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.883576135Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.883622996Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51/snapshots/be250217-78d8-4efa-928c-110459283db3/run","method":"POST","req_id":"4a6def93-6deb-40eb-8b16-7dd93c6e5097","remote_addr":"[fd00:1122:3344:101::a]:38840","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":35850,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.892716947Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.892835867Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.892967538Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.892981738Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.893042539Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.893108979Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.89325618Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.89333555Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.893423481Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.893467491Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.901505176Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51-be250217-78d8-4efa-928c-110459283db3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.904358192Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51-be250217-78d8-4efa-928c-110459283db3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.907764561Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51-be250217-78d8-4efa-928c-110459283db3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.920992865Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51/.zfs/snapshot/be250217-78d8-4efa-928c-110459283db3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.921325587Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.921411027Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.921450518Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.921489588Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.921542798Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.929932595Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.935808628Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"region 92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51 running snapshot be250217-78d8-4efa-928c-110459283db3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:36.935826758Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"path is \"/data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:37.028435237Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"dataset is oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:37.035734538Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"snapshot is oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51@be250217-78d8-4efa-928c-110459283db3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:37.041706341Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51\", \"be250217-78d8-4efa-928c-110459283db3\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:37.041728391Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:37.049357774Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51/snapshots/be250217-78d8-4efa-928c-110459283db3/run","method":"POST","req_id":"180bb401-72b6-4078-871d-e8ec07ced3f3","remote_addr":"[fd00:1122:3344:101::a]:38840","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20977,"response_code":200}
{"msg":"region c7b5024a-2a51-4092-8397-c1e90adef843 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.054827964Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.055023415Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions","method":"POST","req_id":"c72253b9-1e64-4189-b199-73417dbdc459","remote_addr":"[fd00:1122:3344:101::a]:38840","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":245,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.055068345Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/c7b5024a-2a51-4092-8397-c1e90adef843","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.061851183Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/c7b5024a-2a51-4092-8397-c1e90adef843","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.061871014Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c7b5024a-2a51-4092-8397-c1e90adef843\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c7b5024a-2a51-4092-8397-c1e90adef843\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.115068452Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"region":"c7b5024a-2a51-4092-8397-c1e90adef843","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c7b5024a-2a51-4092-8397-c1e90adef843\" \"--data\" \"/data/regions/c7b5024a-2a51-4092-8397-c1e90adef843\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.115153512Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"region":"c7b5024a-2a51-4092-8397-c1e90adef843","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.195840924Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"region":"c7b5024a-2a51-4092-8397-c1e90adef843","component":"worker"}
{"msg":"region c7b5024a-2a51-4092-8397-c1e90adef843 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.195877464Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.195995025Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.196509958Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.196595218Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.196668369Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.196680099Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.196712229Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.196791799Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.197203312Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.197256582Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.197305092Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.197349493Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.201465796Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c7b5024a-2a51-4092-8397-c1e90adef843","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.201832168Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.202419911Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions","method":"POST","req_id":"d35b727c-ad79-4d5a-a5fd-5ec31a9d593b","remote_addr":"[fd00:1122:3344:101::a]:38840","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":48,"response_code":200}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c7b5024a-2a51-4092-8397-c1e90adef843","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.207602Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.219206415Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c7b5024a-2a51-4092-8397-c1e90adef843","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.225040008Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c7b5024a-2a51-4092-8397-c1e90adef843","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.225251439Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.225308909Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.225333779Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.22537255Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.229916655Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:27:42.236775563Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.029487699Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions/c7b5024a-2a51-4092-8397-c1e90adef843","method":"GET","req_id":"9883a854-fc95-4878-9065-6e05a5617166","remote_addr":"[fd00:1122:3344:101::a]:38840","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":71,"response_code":200}
{"msg":"path is \"/data/regions/c7b5024a-2a51-4092-8397-c1e90adef843\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.036255217Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"dataset is oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/c7b5024a-2a51-4092-8397-c1e90adef843","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.045354868Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"region c7b5024a-2a51-4092-8397-c1e90adef843 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.052462108Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.052612398Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.054896371Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions/c7b5024a-2a51-4092-8397-c1e90adef843","method":"DELETE","req_id":"59789c63-c727-4ba9-a7f8-f4eae112fab9","remote_addr":"[fd00:1122:3344:101::a]:38840","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":18670,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-c7b5024a-2a51-4092-8397-c1e90adef843 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.055831717Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.058024529Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions/c7b5024a-2a51-4092-8397-c1e90adef843","method":"GET","req_id":"47f62668-5b4f-40b8-9f84-5dd85261daff","remote_addr":"[fd00:1122:3344:101::a]:38840","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":58,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.061127396Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.061233647Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.061314767Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.061332647Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.061366948Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.061411388Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/92c1d0dd-5aab-4ea4-bbc0-1c08d0e82e51","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.0618199Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.061866841Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.061922621Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.061971891Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.066774288Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.068263776Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_4b90dac0-55a6-4c17-843d-293d33e1305a/crucible/regions/c7b5024a-2a51-4092-8397-c1e90adef843\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.077368387Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"region":"c7b5024a-2a51-4092-8397-c1e90adef843","component":"worker"}
{"msg":"region c7b5024a-2a51-4092-8397-c1e90adef843 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.129491739Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-25T23:28:07.206952003Z","hostname":"oxz_crucible_110ffbc0-cdeb-4d05-99f9-a39567893764","pid":28481,"uri":"/crucible/0/regions/c7b5024a-2a51-4092-8397-c1e90adef843","method":"GET","req_id":"6f2758c4-eb30-4342-b088-b3b66c44b90a","remote_addr":"[fd00:1122:3344:101::a]:38840","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":59,"response_code":200}