[ Nov  9 02:46:17 Enabled. ]
[ Nov  9 02:46:17 Rereading configuration. ]
[ Nov  9 02:46:19 Rereading configuration. ]
[ Nov  9 02:46:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:21.152707374Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:21.155967751Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:21.155983162Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:21.177545797Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:21.30350177Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:12.977853003Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:42975"}
{"msg":"region 1c5c7182-2174-4706-a5c5-80d244ec8bdc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:12.983079722Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:12.983383578Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions","method":"POST","req_id":"f78eda41-235e-4a52-b701-a591d89c8502","remote_addr":"[fd00:1122:3344:101::c]:42975","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":740,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:12.984608673Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:12.994861515Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:12.994890415Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1c5c7182-2174-4706-a5c5-80d244ec8bdc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.061142085Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"1c5c7182-2174-4706-a5c5-80d244ec8bdc","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1c5c7182-2174-4706-a5c5-80d244ec8bdc\" \"--data\" \"/data/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.061260177Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"1c5c7182-2174-4706-a5c5-80d244ec8bdc","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.187293242Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"1c5c7182-2174-4706-a5c5-80d244ec8bdc","component":"worker"}
{"msg":"region 1c5c7182-2174-4706-a5c5-80d244ec8bdc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.187353723Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.187472656Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1c5c7182-2174-4706-a5c5-80d244ec8bdc","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.187925215Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1c5c7182-2174-4706-a5c5-80d244ec8bdc","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.193312726Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.204760673Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1c5c7182-2174-4706-a5c5-80d244ec8bdc","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.208510621Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.209227315Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.209381828Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.20944561Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.209496561Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.213736458Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.21817148Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.279491137Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions","method":"POST","req_id":"e99f772b-d06d-41fa-bbef-3f0456c9e656","remote_addr":"[fd00:1122:3344:101::c]:42975","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.737675036Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc","method":"GET","req_id":"9783b6ce-754b-4fe0-9b7f-372bb04faa28","remote_addr":"[fd00:1122:3344:101::c]:42975","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"path is \"/data/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.741124947Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"dataset is oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.751629794Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"region 1c5c7182-2174-4706-a5c5-80d244ec8bdc state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.761212512Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.761389276Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.761541999Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc","method":"DELETE","req_id":"ec25f4f5-a51c-4ae0-90a5-2cf76a48d439","remote_addr":"[fd00:1122:3344:101::c]:42975","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20445,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-1c5c7182-2174-4706-a5c5-80d244ec8bdc (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.762411807Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.765905199Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc","method":"GET","req_id":"41858a85-b092-4889-959d-d7b3270e0b78","remote_addr":"[fd00:1122:3344:101::c]:42975","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.766586794Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.779519271Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"1c5c7182-2174-4706-a5c5-80d244ec8bdc","component":"worker"}
{"msg":"region 1c5c7182-2174-4706-a5c5-80d244ec8bdc state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.844480893Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.857739567Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions/1c5c7182-2174-4706-a5c5-80d244ec8bdc","method":"GET","req_id":"cfe9e557-7ea1-4959-9945-c51d0b4b1d95","remote_addr":"[fd00:1122:3344:101::c]:42975","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.302413234Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45710"}
{"msg":"region 8f5957ee-33d4-42a7-828f-198b1ec27470 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.302540597Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.303027207Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.303042638Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions","method":"POST","req_id":"3cc25ee4-e708-4f07-a80b-eca54ff0f6e4","remote_addr":"[fd00:1122:3344:101::a]:45710","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":559,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible/regions/8f5957ee-33d4-42a7-828f-198b1ec27470","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.315025305Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible/regions/8f5957ee-33d4-42a7-828f-198b1ec27470","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.315060536Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8f5957ee-33d4-42a7-828f-198b1ec27470\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8f5957ee-33d4-42a7-828f-198b1ec27470\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.38300977Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"8f5957ee-33d4-42a7-828f-198b1ec27470","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8f5957ee-33d4-42a7-828f-198b1ec27470\" \"--data\" \"/data/regions/8f5957ee-33d4-42a7-828f-198b1ec27470\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.384872188Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"8f5957ee-33d4-42a7-828f-198b1ec27470","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.58628562Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions","method":"POST","req_id":"51d1dc3a-19e5-448d-abaa-29322ae70fd3","remote_addr":"[fd00:1122:3344:101::a]:45710","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":852,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.721872382Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"8f5957ee-33d4-42a7-828f-198b1ec27470","component":"worker"}
{"msg":"region 8f5957ee-33d4-42a7-828f-198b1ec27470 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.721914994Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.721994145Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8f5957ee-33d4-42a7-828f-198b1ec27470","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.723373824Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8f5957ee-33d4-42a7-828f-198b1ec27470","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.731343618Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.751911513Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8f5957ee-33d4-42a7-828f-198b1ec27470","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.763041003Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8f5957ee-33d4-42a7-828f-198b1ec27470","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.763275078Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.763659936Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.763665976Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.763673657Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.773374067Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.783079318Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:22.191372925Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions","method":"POST","req_id":"32919892-c741-4148-8a90-bca4046b4c69","remote_addr":"[fd00:1122:3344:101::a]:45710","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"region 90e86b7b-13a9-4f90-86d8-42c73321ec69 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:26.939676003Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:26.939870627Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:26.939891887Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions","method":"POST","req_id":"33f32850-10b4-46e9-b8e2-19cb83459f7a","remote_addr":"[fd00:1122:3344:101::a]:45710","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":268,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:26.949982896Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:26.950017796Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"90e86b7b-13a9-4f90-86d8-42c73321ec69\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.014430608Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"90e86b7b-13a9-4f90-86d8-42c73321ec69","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"90e86b7b-13a9-4f90-86d8-42c73321ec69\" \"--data\" \"/data/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.016288986Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"90e86b7b-13a9-4f90-86d8-42c73321ec69","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.121888138Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions","method":"POST","req_id":"7e861c26-75c1-4085-aa11-34a80d57677b","remote_addr":"[fd00:1122:3344:101::a]:45710","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":137,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.183606134Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"90e86b7b-13a9-4f90-86d8-42c73321ec69","component":"worker"}
{"msg":"region 90e86b7b-13a9-4f90-86d8-42c73321ec69 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.183644815Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.183845169Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-90e86b7b-13a9-4f90-86d8-42c73321ec69","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.186942793Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-90e86b7b-13a9-4f90-86d8-42c73321ec69","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.194703163Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.209456948Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-90e86b7b-13a9-4f90-86d8-42c73321ec69","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.216189137Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.216490624Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.216576005Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.216654107Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.216724048Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.225280945Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.235241591Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:27.458339381Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions","method":"POST","req_id":"2b8880c8-744b-4187-a589-7ff5b229c6be","remote_addr":"[fd00:1122:3344:101::a]:45710","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.184184928Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69","method":"GET","req_id":"b63c05b6-e552-48c2-8568-486c7b5c5ff3","remote_addr":"[fd00:1122:3344:101::a]:45710","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"path is \"/data/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.187952795Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"dataset is oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.202302432Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"region 90e86b7b-13a9-4f90-86d8-42c73321ec69 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.212174046Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.21235395Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.212413001Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69","method":"DELETE","req_id":"a12131eb-4b6b-47d2-bceb-3aa55050ecea","remote_addr":"[fd00:1122:3344:101::a]:45710","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":24499,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-90e86b7b-13a9-4f90-86d8-42c73321ec69 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.214126256Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.215722479Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69","method":"GET","req_id":"fcdaa5ea-5afc-4fe4-96ba-4f11ae9a06d4","remote_addr":"[fd00:1122:3344:101::a]:45710","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":96,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.224208595Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crucible/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.235353225Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"region":"90e86b7b-13a9-4f90-86d8-42c73321ec69","component":"worker"}
{"msg":"region 90e86b7b-13a9-4f90-86d8-42c73321ec69 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.296470598Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:53.330059472Z","hostname":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","pid":28589,"uri":"/crucible/0/regions/90e86b7b-13a9-4f90-86d8-42c73321ec69","method":"GET","req_id":"180325d1-fb20-40b4-8a84-6f63cd5d6c12","remote_addr":"[fd00:1122:3344:101::a]:45710","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":82,"response_code":"200"}