[ Jun  3 01:01:30 Enabled. ]
[ Jun  3 01:01:30 Rereading configuration. ]
[ Jun  3 01:01:31 Rereading configuration. ]
[ Jun  3 01:01:34 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: bd9a0e2abe6b6b89aec8c85f4ee57474144ed150\nCommit timestamp: 2026-05-14T15:48:00.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.906421168Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982}
{"msg":"dataset: \"oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.906597231Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.906613742Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.906620872Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.925013075Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:01:34.998400173Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.664748943Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54391"}
{"msg":"region 2879321a-6aca-49c7-892a-ba56f81b8830 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.699380864Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.704541092Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.705505492Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions","method":"POST","req_id":"9a2f8f2f-76be-4188-aaae-d44edc3f0236","remote_addr":"[fd00:1122:3344:101::c]:54391","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":14188,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.711804163Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.711830844Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2879321a-6aca-49c7-892a-ba56f81b8830\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/2879321a-6aca-49c7-892a-ba56f81b8830\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.787854706Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"region":"2879321a-6aca-49c7-892a-ba56f81b8830","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"2879321a-6aca-49c7-892a-ba56f81b8830\" \"--data\" \"/data/regions/2879321a-6aca-49c7-892a-ba56f81b8830\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.789784077Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"region":"2879321a-6aca-49c7-892a-ba56f81b8830","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.915594466Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"region":"2879321a-6aca-49c7-892a-ba56f81b8830","component":"worker"}
{"msg":"region 2879321a-6aca-49c7-892a-ba56f81b8830 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.915666998Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.915882183Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.918099059Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.922298756Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.933134382Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.934927599Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.935634904Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.935697875Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.935751746Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.935760846Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.94795694Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:18.950861411Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:02:19.047075344Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions","method":"POST","req_id":"5c52b2cc-dd2a-40e4-8427-88fd9b81bfc4","remote_addr":"[fd00:1122:3344:101::c]:54391","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":83,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.637428529Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:38408"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.637529651Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions/2879321a-6aca-49c7-892a-ba56f81b8830","method":"GET","req_id":"38f6817b-4e3d-48a9-96f0-0f73afa71954","remote_addr":"[fd00:1122:3344:101::c]:38408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":55,"response_code":200}
{"msg":"path is \"/data/regions/2879321a-6aca-49c7-892a-ba56f81b8830\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.6422911Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"dataset is oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.64852726Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"snapshot is oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830@c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.654146237Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830\", \"c89e0aa1-5551-4813-9507-8c80561a6b6f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.654168617Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.66338554Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions/2879321a-6aca-49c7-892a-ba56f81b8830/snapshots/c89e0aa1-5551-4813-9507-8c80561a6b6f","method":"GET","req_id":"1d3fdc11-ff26-401f-8ed5-2ac2026a98de","remote_addr":"[fd00:1122:3344:101::c]:38408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":21236,"response_code":200}
{"msg":"path is \"/data/regions/2879321a-6aca-49c7-892a-ba56f81b8830\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.671898617Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"dataset is oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.677568295Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"snapshot is oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830@c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.683308084Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830\", \"c89e0aa1-5551-4813-9507-8c80561a6b6f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.683339105Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 2879321a-6aca-49c7-892a-ba56f81b8830-c89e0aa1-5551-4813-9507-8c80561a6b6f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.69222647Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"applying SMF actions for region 2879321a-6aca-49c7-892a-ba56f81b8830 running snapshot c89e0aa1-5551-4813-9507-8c80561a6b6f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.695313724Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.695340635Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions/2879321a-6aca-49c7-892a-ba56f81b8830/snapshots/c89e0aa1-5551-4813-9507-8c80561a6b6f/run","method":"POST","req_id":"e1b35a16-6681-4f85-b6e1-699a945f0631","remote_addr":"[fd00:1122:3344:101::c]:38408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":23449,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.704492405Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.704566857Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.704641769Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.704648299Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.70468666Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.70473063Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.705046777Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.705097558Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.705145089Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.70519215Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.712058452Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-2879321a-6aca-49c7-892a-ba56f81b8830-c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.71482698Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-2879321a-6aca-49c7-892a-ba56f81b8830-c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.720726183Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-2879321a-6aca-49c7-892a-ba56f81b8830-c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.727483914Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2879321a-6aca-49c7-892a-ba56f81b8830/.zfs/snapshot/c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.727773609Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.727829671Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.727869931Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.727907372Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.727956313Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.730086758Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.733154352Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"region 2879321a-6aca-49c7-892a-ba56f81b8830 running snapshot c89e0aa1-5551-4813-9507-8c80561a6b6f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.733170792Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"path is \"/data/regions/2879321a-6aca-49c7-892a-ba56f81b8830\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.947984885Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"dataset is oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.955254357Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"snapshot is oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830@c89e0aa1-5551-4813-9507-8c80561a6b6f","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.960133428Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/2879321a-6aca-49c7-892a-ba56f81b8830\", \"c89e0aa1-5551-4813-9507-8c80561a6b6f\"]","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.960152799Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:38.96646385Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions/2879321a-6aca-49c7-892a-ba56f81b8830/snapshots/c89e0aa1-5551-4813-9507-8c80561a6b6f/run","method":"POST","req_id":"ddf6f9d1-5898-4d2f-b776-1dae3f07fc91","remote_addr":"[fd00:1122:3344:101::c]:38408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":18519,"response_code":200}
{"msg":"region c27cd86a-67e8-4fde-8748-36f247b2c4c6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.64122471Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.641455195Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions","method":"POST","req_id":"111cee9d-f1eb-4731-b917-84cbcce8ba69","remote_addr":"[fd00:1122:3344:101::c]:38408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":277,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.641484055Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.647670574Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.647687675Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c27cd86a-67e8-4fde-8748-36f247b2c4c6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.691724622Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"region":"c27cd86a-67e8-4fde-8748-36f247b2c4c6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c27cd86a-67e8-4fde-8748-36f247b2c4c6\" \"--data\" \"/data/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.691792673Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"region":"c27cd86a-67e8-4fde-8748-36f247b2c4c6","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.739273542Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"region":"c27cd86a-67e8-4fde-8748-36f247b2c4c6","component":"worker"}
{"msg":"region c27cd86a-67e8-4fde-8748-36f247b2c4c6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.739302312Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.739432235Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.739935525Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.740009157Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.740078618Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.740094019Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.740121269Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.740182381Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.740546408Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.740589449Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.74063856Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.740671131Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.742740134Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c27cd86a-67e8-4fde-8748-36f247b2c4c6","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.743143342Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c27cd86a-67e8-4fde-8748-36f247b2c4c6","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.74640903Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.749702969Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c27cd86a-67e8-4fde-8748-36f247b2c4c6","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.751004966Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.751279121Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.751330753Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.751370694Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.751412444Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.753314334Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.757070132Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:13:43.876602502Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions","method":"POST","req_id":"559569cb-bc47-4252-b8bf-319f9a33f14b","remote_addr":"[fd00:1122:3344:101::c]:38408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":56,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.396312543Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6","method":"GET","req_id":"b9e70f71-1801-4b35-8a19-18293de9abc3","remote_addr":"[fd00:1122:3344:101::c]:38408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":70,"response_code":200}
{"msg":"path is \"/data/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.399588971Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"dataset is oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.410281434Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"region c27cd86a-67e8-4fde-8748-36f247b2c4c6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.417214108Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.417372501Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.417387992Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6","method":"DELETE","req_id":"a9c91d66-e818-4353-b1f5-d1241809bcdb","remote_addr":"[fd00:1122:3344:101::c]:38408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17819,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-c27cd86a-67e8-4fde-8748-36f247b2c4c6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.418003634Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.420745962Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.420820423Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.420895675Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.420907525Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.420942256Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.420998007Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.421231242Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6","method":"GET","req_id":"bec73017-fd04-487c-8782-73ccd70bdd39","remote_addr":"[fd00:1122:3344:101::c]:38408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":49,"response_code":200}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2879321a-6aca-49c7-892a-ba56f81b8830","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.421350904Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.421400545Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.421460126Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.421519218Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.423857066Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.425257145Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0d8fee61-2fbf-4e99-a04f-9a16fae9e41d/crucible/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.433490777Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"region":"c27cd86a-67e8-4fde-8748-36f247b2c4c6","component":"worker"}
{"msg":"region c27cd86a-67e8-4fde-8748-36f247b2c4c6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.475483122Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T01:14:09.536069773Z","hostname":"oxz_crucible_d441ba29-1187-4e6a-8b14-d533db483baa","pid":26982,"uri":"/crucible/0/regions/c27cd86a-67e8-4fde-8748-36f247b2c4c6","method":"GET","req_id":"40982e54-c841-4b81-8110-e546fe59bf54","remote_addr":"[fd00:1122:3344:101::c]:38408","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":58,"response_code":200}