[ Mar  7 22:47:01 Enabled. ]
[ Mar  7 22:47:01 Rereading configuration. ]
[ Mar  7 22:47:05 Rereading configuration. ]
[ Mar  7 22:47:09 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:09.733681665Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:09.736752988Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:09.736771759Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:09.954766183Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.292389724Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:58377"}
{"msg":"region f28383a4-097c-4253-8d99-2393b7dc9edf state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.323276507Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.323625657Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"uri":"/crucible/0/regions","method":"POST","req_id":"dcfa6273-7dc6-49fa-979f-af77c7924774","remote_addr":"[fd00:1122:3344:101::b]:58377","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":1556,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.324352559Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/f28383a4-097c-4253-8d99-2393b7dc9edf","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.33399603Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/f28383a4-097c-4253-8d99-2393b7dc9edf","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.334056002Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f28383a4-097c-4253-8d99-2393b7dc9edf\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/f28383a4-097c-4253-8d99-2393b7dc9edf\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.408539352Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"region":"f28383a4-097c-4253-8d99-2393b7dc9edf","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.472664319Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"uri":"/crucible/0/regions","method":"POST","req_id":"ca06fc32-d502-495a-a2d0-6c96c2743fe3","remote_addr":"[fd00:1122:3344:101::b]:58377","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.543452356Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"region":"f28383a4-097c-4253-8d99-2393b7dc9edf","component":"worker"}
{"msg":"region f28383a4-097c-4253-8d99-2393b7dc9edf state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.543488497Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.543773276Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f28383a4-097c-4253-8d99-2393b7dc9edf","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.54423999Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f28383a4-097c-4253-8d99-2393b7dc9edf","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.551670524Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.566615485Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f28383a4-097c-4253-8d99-2393b7dc9edf","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.573015549Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f28383a4-097c-4253-8d99-2393b7dc9edf","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.57372332Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.573798493Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.573844734Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.573886085Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.579544536Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:30.587372263Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.129686902Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"uri":"/crucible/0/regions","method":"POST","req_id":"7d390d51-00dc-4d31-9382-f70507b64193","remote_addr":"[fd00:1122:3344:101::b]:58377","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":99,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.709158364Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:38705"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.709278727Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"uri":"/crucible/0/regions/f28383a4-097c-4253-8d99-2393b7dc9edf","method":"GET","req_id":"c88cf4f9-8144-46ea-ac62-ae8e98d3bf9f","remote_addr":"[fd00:1122:3344:101::b]:38705","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"path is \"/data/regions/f28383a4-097c-4253-8d99-2393b7dc9edf\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.709437032Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"datafile"}
{"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/f28383a4-097c-4253-8d99-2393b7dc9edf","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.719085043Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"datafile"}
{"msg":"region f28383a4-097c-4253-8d99-2393b7dc9edf state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.729300592Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.729495308Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.72956539Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"uri":"/crucible/0/regions/f28383a4-097c-4253-8d99-2393b7dc9edf","method":"DELETE","req_id":"76bbad95-03da-4a22-a394-7e9a553117b4","remote_addr":"[fd00:1122:3344:101::b]:38705","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20158,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.729697994Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"uri":"/crucible/0/regions/f28383a4-097c-4253-8d99-2393b7dc9edf","method":"GET","req_id":"1d99635c-0920-4eda-8993-deed793ca2f6","remote_addr":"[fd00:1122:3344:101::b]:38705","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-f28383a4-097c-4253-8d99-2393b7dc9edf (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.730640692Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.739454679Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/f28383a4-097c-4253-8d99-2393b7dc9edf\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.748936595Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"region":"f28383a4-097c-4253-8d99-2393b7dc9edf","component":"worker"}
{"msg":"region f28383a4-097c-4253-8d99-2393b7dc9edf state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.812249717Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-07T22:47:31.830022834Z","hostname":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","pid":28691,"uri":"/crucible/0/regions/f28383a4-097c-4253-8d99-2393b7dc9edf","method":"GET","req_id":"b6653794-fcb0-4c09-a214-78c0720c2053","remote_addr":"[fd00:1122:3344:101::b]:38705","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":64,"response_code":"200"}