[ Oct 18 21:17:38 Enabled. ]
[ Oct 18 21:17:38 Rereading configuration. ]
[ Oct 18 21:17:41 Rereading configuration. ]
[ Oct 18 21:17:48 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:17:48.894962332Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:17:48.898995676Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:17:48.899005317Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:17:48.908478745Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:17:49.049792688Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:08.72554723Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59740"}
{"msg":"region 5209d3ff-b32a-44ac-8f5c-03d4afdb10cf state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:08.766933512Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:08.771645476Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions","method":"POST","req_id":"b55df032-7038-483c-9244-1d013f2f7d89","remote_addr":"[fd00:1122:3344:101::a]:59740","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":13970,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:08.776289697Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:08.785370154Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:08.785398595Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5209d3ff-b32a-44ac-8f5c-03d4afdb10cf\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:08.842280586Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"region":"5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5209d3ff-b32a-44ac-8f5c-03d4afdb10cf\" \"--data\" \"/data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:08.847046552Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"region":"5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:08.96983795Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions","method":"POST","req_id":"823f309c-1daa-4837-b8f0-0f0af517a2f3","remote_addr":"[fd00:1122:3344:101::a]:59740","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.063371348Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"region":"5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","component":"worker"}
{"msg":"region 5209d3ff-b32a-44ac-8f5c-03d4afdb10cf state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.0634209Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.063529593Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.066177058Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.070771398Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.091149055Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.104352279Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.104927856Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.104993318Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.105040059Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.1050819Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.110050281Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.117854562Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:19:09.493587485Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions","method":"POST","req_id":"5bd00a23-ef78-4006-8d4a-8050f3d4cb2f","remote_addr":"[fd00:1122:3344:101::a]:59740","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.37086463Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51306"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.374002639Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","method":"GET","req_id":"281cedeb-335c-4c56-b610-98923f8ad2af","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":3092,"response_code":"200"}
{"msg":"path is \"/data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.37651413Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"dataset is oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.385576776Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"snapshot is oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf@ff9fdf84-4f7e-408d-893e-771b2220716d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.394740616Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf\", \"ff9fdf84-4f7e-408d-893e-771b2220716d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.394772347Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.40794631Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf/snapshots/ff9fdf84-4f7e-408d-893e-771b2220716d","method":"GET","req_id":"d02f66e5-0bb0-4a7f-8c08-21e7a470989d","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":31425,"response_code":"200"}
{"msg":"path is \"/data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.411083809Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"dataset is oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.418961222Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"snapshot is oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf@ff9fdf84-4f7e-408d-893e-771b2220716d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.427145254Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf\", \"ff9fdf84-4f7e-408d-893e-771b2220716d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.427179395Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 5209d3ff-b32a-44ac-8f5c-03d4afdb10cf-ff9fdf84-4f7e-408d-893e-771b2220716d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.437055215Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"applying SMF actions for region 5209d3ff-b32a-44ac-8f5c-03d4afdb10cf running snapshot ff9fdf84-4f7e-408d-893e-771b2220716d (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.437821186Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.437845387Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf/snapshots/ff9fdf84-4f7e-408d-893e-771b2220716d/run","method":"POST","req_id":"9a92c397-801c-4646-b44b-d1aba6b78e6c","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":26797,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.445206765Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.445311548Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.445412121Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.445421282Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.445478733Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.445537495Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.445643188Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.4457053Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.445753431Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.445803333Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.452113521Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-5209d3ff-b32a-44ac-8f5c-03d4afdb10cf-ff9fdf84-4f7e-408d-893e-771b2220716d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.452571974Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-5209d3ff-b32a-44ac-8f5c-03d4afdb10cf-ff9fdf84-4f7e-408d-893e-771b2220716d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.455631581Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-5209d3ff-b32a-44ac-8f5c-03d4afdb10cf-ff9fdf84-4f7e-408d-893e-771b2220716d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.466442077Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf/.zfs/snapshot/ff9fdf84-4f7e-408d-893e-771b2220716d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.466648453Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.466709915Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.466757616Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.466804537Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.466865019Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.471378297Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.475742701Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"region 5209d3ff-b32a-44ac-8f5c-03d4afdb10cf running snapshot ff9fdf84-4f7e-408d-893e-771b2220716d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.475760021Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"path is \"/data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.806054897Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"dataset is oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.815801883Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"snapshot is oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf@ff9fdf84-4f7e-408d-893e-771b2220716d","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.824281564Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf\", \"ff9fdf84-4f7e-408d-893e-771b2220716d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.824317355Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:17.832865567Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf/snapshots/ff9fdf84-4f7e-408d-893e-771b2220716d/run","method":"POST","req_id":"42b226f9-1239-42cc-862e-661b4dcc2638","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":26863,"response_code":"200"}
{"msg":"region 9d7bde37-5684-4cee-8e24-e3c4f09866a8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:23.526037346Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:23.526226581Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions","method":"POST","req_id":"4c228724-5591-4040-a2c8-8494a9e82e92","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":252,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:23.526256773Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:23.535897835Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:23.535928006Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9d7bde37-5684-4cee-8e24-e3c4f09866a8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:23.715285677Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"region":"9d7bde37-5684-4cee-8e24-e3c4f09866a8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9d7bde37-5684-4cee-8e24-e3c4f09866a8\" \"--data\" \"/data/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:23.717126159Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"region":"9d7bde37-5684-4cee-8e24-e3c4f09866a8","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:23.862699313Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions","method":"POST","req_id":"478f797b-9be2-4f45-a3b5-cd90c9feb8b6","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.014308587Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"region":"9d7bde37-5684-4cee-8e24-e3c4f09866a8","component":"worker"}
{"msg":"region 9d7bde37-5684-4cee-8e24-e3c4f09866a8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.014339008Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.014484232Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.015123001Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.015211993Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.015294975Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.015316476Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.015344927Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.015395828Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.015779109Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.015841591Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.015891112Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.015942264Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.077145007Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9d7bde37-5684-4cee-8e24-e3c4f09866a8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.077648182Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9d7bde37-5684-4cee-8e24-e3c4f09866a8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.110144542Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.174068113Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.198619919Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions","method":"POST","req_id":"a7a1146c-adeb-4017-b98e-ed6fc6bb1564","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9d7bde37-5684-4cee-8e24-e3c4f09866a8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.223839323Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.224275475Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.224387158Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.224473061Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.224566894Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.264937377Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:24.303190921Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.492715623Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8","method":"GET","req_id":"1c4a56f3-3030-41a7-817b-9a1dc8c15af3","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":100,"response_code":"200"}
{"msg":"path is \"/data/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.495988946Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"dataset is oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.50956891Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"region 9d7bde37-5684-4cee-8e24-e3c4f09866a8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.520913552Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.521103327Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-9d7bde37-5684-4cee-8e24-e3c4f09866a8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.522372293Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.523735141Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8","method":"DELETE","req_id":"e302329c-9d27-4811-bdb7-8177b816600c","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":27775,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.527533389Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8","method":"GET","req_id":"85b6342b-df57-4217-b8d9-84b7e4ec1d81","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":572,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.528171247Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.52826109Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.528346702Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.528359123Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.528394393Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.528442955Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5209d3ff-b32a-44ac-8f5c-03d4afdb10cf","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.528760414Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.528823245Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.528874827Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.528920728Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.534784745Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.536219115Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_92e9deaa-2b6f-4faa-8530-0068451e5459/crucible/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.547852675Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"region":"9d7bde37-5684-4cee-8e24-e3c4f09866a8","component":"worker"}
{"msg":"region 9d7bde37-5684-4cee-8e24-e3c4f09866a8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.620314027Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-18T21:22:54.663963094Z","hostname":"oxz_crucible_1c570b08-efa9-4df4-86ae-968c8a663eef","pid":22506,"uri":"/crucible/0/regions/9d7bde37-5684-4cee-8e24-e3c4f09866a8","method":"GET","req_id":"47b686a7-74ce-4c78-afe4-581ddca9ea9a","remote_addr":"[fd00:1122:3344:101::a]:51306","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":78,"response_code":"200"}