[ May 15 04:12:45 Enabled. ]
[ May 15 04:12:45 Rereading configuration. ]
[ May 15 04:12:47 Rereading configuration. ]
[ May 15 04:12:52 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f9de2043-47b2-4be8-ac5e-48b3cba8acce/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f9de2043-47b2-4be8-ac5e-48b3cba8acce/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:12:52.167378315Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:12:52.170422622Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:12:52.170430972Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:12:52.185835018Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:12:52.360463963Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:24.915455696Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:43865"}
{"msg":"region 6c974831-2b27-497f-bccd-00b6306a4aff state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:24.936569063Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:24.941416986Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"uri":"/crucible/0/regions","method":"POST","req_id":"11c93c8c-cb18-4a2b-a8a7-784c8baf106c","remote_addr":"[fd00:1122:3344:101::c]:43865","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":16495,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:24.944028779Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_f9de2043-47b2-4be8-ac5e-48b3cba8acce/crucible/regions/6c974831-2b27-497f-bccd-00b6306a4aff","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:24.953016533Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_f9de2043-47b2-4be8-ac5e-48b3cba8acce/crucible/regions/6c974831-2b27-497f-bccd-00b6306a4aff","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:24.953041293Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6c974831-2b27-497f-bccd-00b6306a4aff\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6c974831-2b27-497f-bccd-00b6306a4aff\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.011376776Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"region":"6c974831-2b27-497f-bccd-00b6306a4aff","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6c974831-2b27-497f-bccd-00b6306a4aff\" \"--data\" \"/data/regions/6c974831-2b27-497f-bccd-00b6306a4aff\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.011432008Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"region":"6c974831-2b27-497f-bccd-00b6306a4aff","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.079554969Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"uri":"/crucible/0/regions","method":"POST","req_id":"3ab4d169-368b-43fb-96ed-419e217a65ed","remote_addr":"[fd00:1122:3344:101::c]:43865","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.302986806Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"region":"6c974831-2b27-497f-bccd-00b6306a4aff","component":"worker"}
{"msg":"region 6c974831-2b27-497f-bccd-00b6306a4aff state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.303060158Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.303342627Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6c974831-2b27-497f-bccd-00b6306a4aff","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.30374967Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6c974831-2b27-497f-bccd-00b6306a4aff","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.308891893Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.326120226Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c974831-2b27-497f-bccd-00b6306a4aff","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.330301759Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c974831-2b27-497f-bccd-00b6306a4aff","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.331036182Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.331098434Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.331153985Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.331188177Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.339387766Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.343885238Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.389761257Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"uri":"/crucible/0/regions","method":"POST","req_id":"713c3d0e-da5f-47de-8d88-841006119977","remote_addr":"[fd00:1122:3344:101::c]:43865","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":48,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.908246963Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"uri":"/crucible/0/regions/6c974831-2b27-497f-bccd-00b6306a4aff","method":"GET","req_id":"e43247a8-8d26-4428-a868-254b9ee40fd0","remote_addr":"[fd00:1122:3344:101::c]:43865","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"path is \"/data/regions/6c974831-2b27-497f-bccd-00b6306a4aff\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.911073982Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"datafile"}
{"msg":"dataset is oxp_f9de2043-47b2-4be8-ac5e-48b3cba8acce/crucible/regions/6c974831-2b27-497f-bccd-00b6306a4aff","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.922578216Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"datafile"}
{"msg":"region 6c974831-2b27-497f-bccd-00b6306a4aff state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.932992875Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.93314983Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.933221242Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"uri":"/crucible/0/regions/6c974831-2b27-497f-bccd-00b6306a4aff","method":"DELETE","req_id":"0c21bad8-dc80-492e-be2c-70dd5f034256","remote_addr":"[fd00:1122:3344:101::c]:43865","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":22179,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-6c974831-2b27-497f-bccd-00b6306a4aff (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.934223153Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.935520405Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"uri":"/crucible/0/regions/6c974831-2b27-497f-bccd-00b6306a4aff","method":"GET","req_id":"9fe15264-9fe6-4eab-acef-00eca85da8eb","remote_addr":"[fd00:1122:3344:101::c]:43865","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.9385362Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f9de2043-47b2-4be8-ac5e-48b3cba8acce/crucible/regions/6c974831-2b27-497f-bccd-00b6306a4aff\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:25.94993857Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"region":"6c974831-2b27-497f-bccd-00b6306a4aff","component":"worker"}
{"msg":"region 6c974831-2b27-497f-bccd-00b6306a4aff state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:26.016496512Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:26.037444474Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"uri":"/crucible/0/regions/6c974831-2b27-497f-bccd-00b6306a4aff","method":"GET","req_id":"537d76db-0807-4c35-90c1-d21107c61011","remote_addr":"[fd00:1122:3344:101::c]:43865","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"region 51aea369-a278-45d5-bb03-80a11e3d8ec4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:27.872686478Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:27.872830183Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:27.872859954Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"uri":"/crucible/0/regions","method":"POST","req_id":"3cb51823-a9b0-4116-9f3d-9c5cf80f1997","remote_addr":"[fd00:1122:3344:101::c]:43865","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":210,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_f9de2043-47b2-4be8-ac5e-48b3cba8acce/crucible/regions/51aea369-a278-45d5-bb03-80a11e3d8ec4","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:27.882662573Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f9de2043-47b2-4be8-ac5e-48b3cba8acce/crucible/regions/51aea369-a278-45d5-bb03-80a11e3d8ec4","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:27.882685404Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"51aea369-a278-45d5-bb03-80a11e3d8ec4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/51aea369-a278-45d5-bb03-80a11e3d8ec4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:27.927944054Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"region":"51aea369-a278-45d5-bb03-80a11e3d8ec4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"51aea369-a278-45d5-bb03-80a11e3d8ec4\" \"--data\" \"/data/regions/51aea369-a278-45d5-bb03-80a11e3d8ec4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:27.928003746Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"region":"51aea369-a278-45d5-bb03-80a11e3d8ec4","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.006793294Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"region":"51aea369-a278-45d5-bb03-80a11e3d8ec4","component":"worker"}
{"msg":"region 51aea369-a278-45d5-bb03-80a11e3d8ec4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.006855916Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.006931158Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-51aea369-a278-45d5-bb03-80a11e3d8ec4","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.008338263Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-51aea369-a278-45d5-bb03-80a11e3d8ec4","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.0114155Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.018563036Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-51aea369-a278-45d5-bb03-80a11e3d8ec4","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.02152471Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/51aea369-a278-45d5-bb03-80a11e3d8ec4","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.021733256Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.021788078Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.021830009Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.02186971Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.02565823Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.028615954Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"uri":"/crucible/0/regions","method":"POST","req_id":"8c43477d-d8d9-47dd-be84-e460bd47a697","remote_addr":"[fd00:1122:3344:101::c]:43865","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-15T04:13:28.030473552Z","hostname":"oxz_crucible_bd8012e9-d047-4cda-963b-9be0eee4d434","pid":23914,"component":"worker"}