[ Apr  3 22:22:25 Enabled. ]
[ Apr  3 22:22:25 Rereading configuration. ]
[ Apr  3 22:22:27 Rereading configuration. ]
[ Apr  3 22:22:29 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:22:29.487636618Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:22:29.48805293Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:22:29.4880661Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:22:29.725883549Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.508550508Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42343"}
{"msg":"region 66a22c32-6099-4cc5-b3b2-fc4cf9b96793 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.576368626Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.581516019Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.58154549Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"uri":"/crucible/0/regions","method":"POST","req_id":"ecb881e2-a921-43ee-88ea-e5b1ceb8ce64","remote_addr":"[fd00:1122:3344:101::a]:42343","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":12144,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8/crucible/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.595322663Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8/crucible/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.595363704Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"66a22c32-6099-4cc5-b3b2-fc4cf9b96793\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.653102351Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"region":"66a22c32-6099-4cc5-b3b2-fc4cf9b96793","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.833930574Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"uri":"/crucible/0/regions","method":"POST","req_id":"45c42745-75b5-49f6-a342-bb391369b6f3","remote_addr":"[fd00:1122:3344:101::a]:42343","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":111,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.933903217Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"region":"66a22c32-6099-4cc5-b3b2-fc4cf9b96793","component":"worker"}
{"msg":"region 66a22c32-6099-4cc5-b3b2-fc4cf9b96793 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.933941847Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.934071391Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-66a22c32-6099-4cc5-b3b2-fc4cf9b96793","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.934520094Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-66a22c32-6099-4cc5-b3b2-fc4cf9b96793","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.938941927Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.95740184Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-66a22c32-6099-4cc5-b3b2-fc4cf9b96793","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.960379923Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.961073833Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.963323335Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.963374427Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.963414938Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.972855371Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:11.979568857Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:12.097452669Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"uri":"/crucible/0/regions","method":"POST","req_id":"5e8f9aef-2981-4627-b61e-008ced77e224","remote_addr":"[fd00:1122:3344:101::a]:42343","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.501838993Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:38977"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.501926976Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"uri":"/crucible/0/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793","method":"GET","req_id":"d74eca83-5199-4cbc-8655-e66d440b6612","remote_addr":"[fd00:1122:3344:101::a]:38977","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"path is \"/data/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.502113661Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"datafile"}
{"msg":"dataset is oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8/crucible/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.518027664Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"datafile"}
{"msg":"region 66a22c32-6099-4cc5-b3b2-fc4cf9b96793 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.527343363Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.527531468Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.52759983Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"uri":"/crucible/0/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793","method":"DELETE","req_id":"33d31192-cdf6-4582-9c39-33b71be32185","remote_addr":"[fd00:1122:3344:101::a]:38977","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25548,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.527745305Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"uri":"/crucible/0/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793","method":"GET","req_id":"e8d5b3c0-fe73-40e6-9f39-9882181c71b9","remote_addr":"[fd00:1122:3344:101::a]:38977","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-66a22c32-6099-4cc5-b3b2-fc4cf9b96793 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.528596568Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.553282465Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_00b9d2d0-59a4-4682-aa18-072c6202c0c8/crucible/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793\"","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.562953704Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"region":"66a22c32-6099-4cc5-b3b2-fc4cf9b96793","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.615111796Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"uri":"/crucible/0/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793","method":"GET","req_id":"1e50422b-6862-45a4-abba-8a6e2c650432","remote_addr":"[fd00:1122:3344:101::a]:38977","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region 66a22c32-6099-4cc5-b3b2-fc4cf9b96793 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.627593313Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-04-03T22:29:35.687290165Z","hostname":"oxz_crucible_5e26c358-ec8d-4c32-994c-352789460794","pid":23307,"uri":"/crucible/0/regions/66a22c32-6099-4cc5-b3b2-fc4cf9b96793","method":"GET","req_id":"bd7b3044-8116-4432-8bf8-93157cad0a4d","remote_addr":"[fd00:1122:3344:101::a]:38977","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":18,"response_code":"200"}