[ Sep  9 23:01:13 Enabled. ]
[ Sep  9 23:01:13 Rereading configuration. ]
[ Sep  9 23:01:16 Rereading configuration. ]
[ Sep  9 23:01:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 779775d5130ff7a4836f52f48b7e64d1479ee104\nCommit timestamp: 2025-08-09T00:27:59.000000000Z  branch: main\nrustc: 1.86.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:01:21.10565468Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205}
{"msg":"dataset: \"oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:01:21.108600262Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:01:21.108623562Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:01:21.108630612Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:01:21.115861349Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:01:21.22303309Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.085675073Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:33562"}
{"msg":"region 7ef38a0f-ac7e-473c-84a3-06f6e8d739ca state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.137738787Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.1430168Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions","method":"POST","req_id":"341ec4e3-4a61-440a-ab45-1401ea0b0eff","remote_addr":"[fd00:1122:3344:101::b]:33562","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27077,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.148407394Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.157516514Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.157545175Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"7ef38a0f-ac7e-473c-84a3-06f6e8d739ca\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.202713507Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"region":"7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7ef38a0f-ac7e-473c-84a3-06f6e8d739ca\" \"--data\" \"/data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.204972797Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"region":"7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.497881777Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions","method":"POST","req_id":"ee9fcc34-a0d5-467a-9103-16af5d1a5651","remote_addr":"[fd00:1122:3344:101::b]:33562","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":65,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.52313926Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"region":"7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","component":"worker"}
{"msg":"region 7ef38a0f-ac7e-473c-84a3-06f6e8d739ca state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.523175541Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.523316253Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.523767241Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.528750139Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.541051475Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.543695431Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.544415494Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.553491303Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.553571274Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.553604595Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.55673114Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.56076474Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:02:13.87759642Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions","method":"POST","req_id":"ee05c52a-3040-4714-ab8b-2e5596828486","remote_addr":"[fd00:1122:3344:101::b]:33562","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":46,"response_code":200}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.278477739Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:39108"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.278763704Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","method":"GET","req_id":"d0d59464-3ee9-44c9-9873-9c0211608c9a","remote_addr":"[fd00:1122:3344:101::b]:39108","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":53,"response_code":200}
{"msg":"path is \"/data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.284629047Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"dataset is oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.294691353Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"snapshot is oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca@b558694f-ee91-4483-b7e5-fc47938cce3d","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.303989906Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca\", \"b558694f-ee91-4483-b7e5-fc47938cce3d\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.304034617Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.318878548Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca/snapshots/b558694f-ee91-4483-b7e5-fc47938cce3d","method":"GET","req_id":"424342c0-6e47-450a-a7ad-8185abb3da47","remote_addr":"[fd00:1122:3344:101::b]:39108","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":37107,"response_code":200}
{"msg":"path is \"/data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.324282972Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"dataset is oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.332734661Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"snapshot is oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca@b558694f-ee91-4483-b7e5-fc47938cce3d","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.340129541Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca\", \"b558694f-ee91-4483-b7e5-fc47938cce3d\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.340171641Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 7ef38a0f-ac7e-473c-84a3-06f6e8d739ca-b558694f-ee91-4483-b7e5-fc47938cce3d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.348353975Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"applying SMF actions for region 7ef38a0f-ac7e-473c-84a3-06f6e8d739ca running snapshot b558694f-ee91-4483-b7e5-fc47938cce3d (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.348898244Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.348922525Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca/snapshots/b558694f-ee91-4483-b7e5-fc47938cce3d/run","method":"POST","req_id":"375a7f8e-7dd1-4aad-8d6b-c2a4a45a2874","remote_addr":"[fd00:1122:3344:101::b]:39108","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24670,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.368358866Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.368459787Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.368531499Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.368539739Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.36857593Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.36861672Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.368716842Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.368773743Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.368815044Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.368857965Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.378069676Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-7ef38a0f-ac7e-473c-84a3-06f6e8d739ca-b558694f-ee91-4483-b7e5-fc47938cce3d","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.378530554Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-7ef38a0f-ac7e-473c-84a3-06f6e8d739ca-b558694f-ee91-4483-b7e5-fc47938cce3d","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.385554078Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-7ef38a0f-ac7e-473c-84a3-06f6e8d739ca-b558694f-ee91-4483-b7e5-fc47938cce3d","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.397234963Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca/.zfs/snapshot/b558694f-ee91-4483-b7e5-fc47938cce3d","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.397527018Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.397582819Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.397620029Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.39765563Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.397704031Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.401252973Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.405469357Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"region 7ef38a0f-ac7e-473c-84a3-06f6e8d739ca running snapshot b558694f-ee91-4483-b7e5-fc47938cce3d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.405482237Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"path is \"/data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.649235314Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"dataset is oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.659659217Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"snapshot is oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca@b558694f-ee91-4483-b7e5-fc47938cce3d","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.668660435Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca\", \"b558694f-ee91-4483-b7e5-fc47938cce3d\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.668719126Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:28.676607834Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca/snapshots/b558694f-ee91-4483-b7e5-fc47938cce3d/run","method":"POST","req_id":"de2a4762-d490-4f98-81bd-e904579d1001","remote_addr":"[fd00:1122:3344:101::b]:39108","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27447,"response_code":200}
{"msg":"region 32db2b40-67b7-4291-8144-dc4de5852aff state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:33.933877193Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:33.934066296Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions","method":"POST","req_id":"1e9e74a8-0626-4400-b8f8-3206439a8188","remote_addr":"[fd00:1122:3344:101::b]:39108","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":240,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:33.934091957Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/32db2b40-67b7-4291-8144-dc4de5852aff","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:33.944960397Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/32db2b40-67b7-4291-8144-dc4de5852aff","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:33.944999438Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"32db2b40-67b7-4291-8144-dc4de5852aff\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/32db2b40-67b7-4291-8144-dc4de5852aff\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.005289706Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"region":"32db2b40-67b7-4291-8144-dc4de5852aff","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"32db2b40-67b7-4291-8144-dc4de5852aff\" \"--data\" \"/data/regions/32db2b40-67b7-4291-8144-dc4de5852aff\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.007681168Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"region":"32db2b40-67b7-4291-8144-dc4de5852aff","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.107291136Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"region":"32db2b40-67b7-4291-8144-dc4de5852aff","component":"worker"}
{"msg":"region 32db2b40-67b7-4291-8144-dc4de5852aff state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.107321016Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.107442428Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-32db2b40-67b7-4291-8144-dc4de5852aff","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.107850346Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-32db2b40-67b7-4291-8144-dc4de5852aff","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.110887539Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.117906862Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-32db2b40-67b7-4291-8144-dc4de5852aff","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.120871684Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/32db2b40-67b7-4291-8144-dc4de5852aff","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.121161679Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.12121409Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.121252781Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.121288711Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.125323812Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.129796671Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.129916333Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.130019484Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.130038605Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.130064475Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.130107286Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.133606818Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.133655728Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.133698519Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.13374094Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.137328733Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.139482681Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:34.311505569Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions","method":"POST","req_id":"0b697b81-101a-4654-8bf2-ab2a288ebed3","remote_addr":"[fd00:1122:3344:101::b]:39108","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":60,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.817686974Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions/32db2b40-67b7-4291-8144-dc4de5852aff","method":"GET","req_id":"a3015671-6d57-436a-8b9c-56fac9d81f21","remote_addr":"[fd00:1122:3344:101::b]:39108","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":52,"response_code":200}
{"msg":"path is \"/data/regions/32db2b40-67b7-4291-8144-dc4de5852aff\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.823059158Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"dataset is oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/32db2b40-67b7-4291-8144-dc4de5852aff","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.837194336Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"region 32db2b40-67b7-4291-8144-dc4de5852aff state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.847462156Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.84766936Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.847723941Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions/32db2b40-67b7-4291-8144-dc4de5852aff","method":"DELETE","req_id":"708ffefc-d7ec-49c1-8b34-d976ddd9b98d","remote_addr":"[fd00:1122:3344:101::b]:39108","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27110,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-32db2b40-67b7-4291-8144-dc4de5852aff (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.848701548Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.850556941Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions/32db2b40-67b7-4291-8144-dc4de5852aff","method":"GET","req_id":"9fc0ed10-198f-4a97-af56-ae0863cde7f7","remote_addr":"[fd00:1122:3344:101::b]:39108","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":37,"response_code":200}
{"msg":"existing directory value /dev/null does not match /data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.855178352Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.855256923Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.855331455Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.855343145Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.855373765Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.855413406Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7ef38a0f-ac7e-473c-84a3-06f6e8d739ca","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.855749552Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.855806463Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.855861704Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.855887734Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.859190402Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.860452415Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6e8ee60e-15af-4b7a-bbf0-13c66a9dfadf/crucible/regions/32db2b40-67b7-4291-8144-dc4de5852aff\"","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.872677939Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"region":"32db2b40-67b7-4291-8144-dc4de5852aff","component":"worker"}
{"msg":"region 32db2b40-67b7-4291-8144-dc4de5852aff state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.939908939Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-09-09T23:12:58.99466885Z","hostname":"oxz_crucible_a1124e4a-8431-411d-bf72-42613bf2aa87","pid":25205,"uri":"/crucible/0/regions/32db2b40-67b7-4291-8144-dc4de5852aff","method":"GET","req_id":"f967e4c2-a64c-4952-b813-99a67bf0a4e2","remote_addr":"[fd00:1122:3344:101::b]:39108","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":67,"response_code":200}