[ Jan 18 05:33:09 Enabled. ]
[ Jan 18 05:33:09 Rereading configuration. ]
[ Jan 18 05:33:15 Rereading configuration. ]
[ Jan 18 05:33:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:33:21.743501525Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:33:21.74449499Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:33:21.74450552Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:33:21.762804983Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:33:21.857744829Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.396205743Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:61453"}
{"msg":"region 72566508-5a2a-4dfe-9ef8-d43a85c89b85 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.427852975Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.432507197Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions","method":"POST","req_id":"64237e41-9549-49f5-b935-894cf0c7f089","remote_addr":"[fd00:1122:3344:101::b]:61453","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":11403,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.435157214Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.444728677Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.444763198Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"72566508-5a2a-4dfe-9ef8-d43a85c89b85\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.492691173Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"region":"72566508-5a2a-4dfe-9ef8-d43a85c89b85","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"72566508-5a2a-4dfe-9ef8-d43a85c89b85\" \"--data\" \"/data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.494622771Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"region":"72566508-5a2a-4dfe-9ef8-d43a85c89b85","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.668035999Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions","method":"POST","req_id":"092666be-5768-4553-94b4-376c075d656f","remote_addr":"[fd00:1122:3344:101::b]:61453","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":90,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.688355122Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"region":"72566508-5a2a-4dfe-9ef8-d43a85c89b85","component":"worker"}
{"msg":"region 72566508-5a2a-4dfe-9ef8-d43a85c89b85 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.688384962Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.688494074Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.68887797Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.694294101Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.703802853Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.714002795Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.714509521Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.716486392Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.716548643Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.716596874Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.72035567Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:10.724671034Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:34:11.185022132Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions","method":"POST","req_id":"87c5c192-89b0-4068-a0ec-27787eb83b20","remote_addr":"[fd00:1122:3344:101::b]:61453","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.694955747Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:36002"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.695067649Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","method":"GET","req_id":"0b04fa0a-ff60-4416-b9cb-ab22f446ab55","remote_addr":"[fd00:1122:3344:101::b]:36002","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"path is \"/data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.698678713Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"dataset is oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.707612926Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"snapshot is oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85@55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.715457703Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85\", \"55708258-8abd-4c4b-8508-8aec30cbe904\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.715485173Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.727758197Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85/snapshots/55708258-8abd-4c4b-8508-8aec30cbe904","method":"GET","req_id":"f50d0ffd-efb8-49fe-b308-5f64f9ec538e","remote_addr":"[fd00:1122:3344:101::b]:36002","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":29084,"response_code":"200"}
{"msg":"path is \"/data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.731064356Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"dataset is oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.739869897Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"snapshot is oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85@55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.74741609Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85\", \"55708258-8abd-4c4b-8508-8aec30cbe904\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.74744331Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 72566508-5a2a-4dfe-9ef8-d43a85c89b85-55708258-8abd-4c4b-8508-8aec30cbe904 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.757342598Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"applying SMF actions for region 72566508-5a2a-4dfe-9ef8-d43a85c89b85 running snapshot 55708258-8abd-4c4b-8508-8aec30cbe904 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.757953467Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.757987508Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85/snapshots/55708258-8abd-4c4b-8508-8aec30cbe904/run","method":"POST","req_id":"1dae0a0b-f70a-4d56-b903-7e2200113a5e","remote_addr":"[fd00:1122:3344:101::b]:36002","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":26962,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.76216713Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.762252111Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.762333872Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.762340983Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.762382093Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.762429584Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.762525615Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.762587706Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.762635137Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.762684308Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.76751779Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-72566508-5a2a-4dfe-9ef8-d43a85c89b85-55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.767967497Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-72566508-5a2a-4dfe-9ef8-d43a85c89b85-55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.770519045Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-72566508-5a2a-4dfe-9ef8-d43a85c89b85-55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.77958173Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85/.zfs/snapshot/55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.779853834Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.779921695Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.779962666Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.780001016Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.780057777Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.783296115Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.787498738Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"region 72566508-5a2a-4dfe-9ef8-d43a85c89b85 running snapshot 55708258-8abd-4c4b-8508-8aec30cbe904 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:09.787510398Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"path is \"/data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:10.008737229Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"dataset is oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:10.017925346Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"snapshot is oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85@55708258-8abd-4c4b-8508-8aec30cbe904","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:10.026988811Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85\", \"55708258-8abd-4c4b-8508-8aec30cbe904\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:10.027018421Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:10.034517513Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85/snapshots/55708258-8abd-4c4b-8508-8aec30cbe904/run","method":"POST","req_id":"40c9cd3c-6f77-41c5-8a17-0fd9772d5da3","remote_addr":"[fd00:1122:3344:101::b]:36002","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25825,"response_code":"200"}
{"msg":"region f2a3df49-dceb-47f4-b856-9d92278245a5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.419539601Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.419717063Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions","method":"POST","req_id":"debbd030-c453-4145-b51a-1fa2ad308312","remote_addr":"[fd00:1122:3344:101::b]:36002","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":242,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.419744234Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/f2a3df49-dceb-47f4-b856-9d92278245a5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.429956246Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/f2a3df49-dceb-47f4-b856-9d92278245a5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.429982497Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f2a3df49-dceb-47f4-b856-9d92278245a5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f2a3df49-dceb-47f4-b856-9d92278245a5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.491101919Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"region":"f2a3df49-dceb-47f4-b856-9d92278245a5","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f2a3df49-dceb-47f4-b856-9d92278245a5\" \"--data\" \"/data/regions/f2a3df49-dceb-47f4-b856-9d92278245a5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.492928376Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"region":"f2a3df49-dceb-47f4-b856-9d92278245a5","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.560714787Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"region":"f2a3df49-dceb-47f4-b856-9d92278245a5","component":"worker"}
{"msg":"region f2a3df49-dceb-47f4-b856-9d92278245a5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.560755368Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.56088228Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.561680222Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.561707552Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.561782213Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.561812144Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.561833044Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.561902215Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.562369982Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.562508124Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.562609086Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.562655906Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.566743557Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f2a3df49-dceb-47f4-b856-9d92278245a5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.567296376Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f2a3df49-dceb-47f4-b856-9d92278245a5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.579170083Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.58700981Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f2a3df49-dceb-47f4-b856-9d92278245a5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.590611493Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f2a3df49-dceb-47f4-b856-9d92278245a5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.590874067Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.590942429Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.590993209Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.59104525Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.594610183Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.601376624Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:13.643825218Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions","method":"POST","req_id":"adf840b2-5cb2-4a21-8856-cd412fea5911","remote_addr":"[fd00:1122:3344:101::b]:36002","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.102519971Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions/f2a3df49-dceb-47f4-b856-9d92278245a5","method":"GET","req_id":"add9b0da-3595-4676-a55d-c5b9cc091f14","remote_addr":"[fd00:1122:3344:101::b]:36002","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"path is \"/data/regions/f2a3df49-dceb-47f4-b856-9d92278245a5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.105944072Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"dataset is oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/f2a3df49-dceb-47f4-b856-9d92278245a5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.118178115Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"region f2a3df49-dceb-47f4-b856-9d92278245a5 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.128132323Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.128348366Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-f2a3df49-dceb-47f4-b856-9d92278245a5 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.129643195Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.13127448Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions/f2a3df49-dceb-47f4-b856-9d92278245a5","method":"DELETE","req_id":"16df8626-09f5-447c-a030-2471db7f8a20","remote_addr":"[fd00:1122:3344:101::b]:36002","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25361,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.133900099Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions/f2a3df49-dceb-47f4-b856-9d92278245a5","method":"GET","req_id":"d9d59ac7-ad07-466b-996d-d0959586d93b","remote_addr":"[fd00:1122:3344:101::b]:36002","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":45,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.133926329Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.133995811Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134092972Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134106692Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134145333Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134199243Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/72566508-5a2a-4dfe-9ef8-d43a85c89b85","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134521238Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134590569Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.13463607Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.134683741Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.138267494Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.139631584Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_bd4f6a2e-c7eb-469f-bd80-c35295e612ed/crucible/regions/f2a3df49-dceb-47f4-b856-9d92278245a5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.151156356Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"region":"f2a3df49-dceb-47f4-b856-9d92278245a5","component":"worker"}
{"msg":"region f2a3df49-dceb-47f4-b856-9d92278245a5 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.204897688Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-18T05:44:38.226885426Z","hostname":"oxz_crucible_01a4829a-1f64-47ac-a2b0-622ca42defd2","pid":22336,"uri":"/crucible/0/regions/f2a3df49-dceb-47f4-b856-9d92278245a5","method":"GET","req_id":"1d62adc8-8838-40bd-a8c7-9deff633fe07","remote_addr":"[fd00:1122:3344:101::b]:36002","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":85,"response_code":"200"}