[ Mar 21 18:03:38 Enabled. ]
[ Mar 21 18:03:38 Rereading configuration. ]
[ Mar 21 18:03:39 Rereading configuration. ]
[ Mar 21 18:03:42 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:42.543450702Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:42.546862883Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:42.546886844Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:42.775237876Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.081028749Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64965"}
{"msg":"region 9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.116023796Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.116309352Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions","method":"POST","req_id":"17e96663-d6d3-4a12-b2a5-9f4a9d31ae8a","remote_addr":"[fd00:1122:3344:101::a]:64965","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":5234,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.117057405Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.126995793Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.127032114Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.207658149Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"region":"9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.328564715Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"region":"9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","component":"worker"}
{"msg":"region 9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.328613429Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.328695892Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.329119613Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.33607353Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.353447982Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.358498592Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.359230805Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.361330653Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.361385354Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.361431385Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.371113008Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.378222686Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.470416707Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions","method":"POST","req_id":"8eb24f18-40bc-4822-b0c3-004b6774a230","remote_addr":"[fd00:1122:3344:101::a]:64965","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.892655605Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:41406"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.893388438Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","method":"GET","req_id":"a2fe6425-5423-4979-9d5e-0a6f7707d0a3","remote_addr":"[fd00:1122:3344:101::a]:41406","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":719,"response_code":"200"}
{"msg":"path is \"/data/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.893550431Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.902864438Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"region 9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.912288137Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.91245129Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.912515991Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","method":"DELETE","req_id":"d5e8a0ee-8c61-47cb-9928-b80df0666861","remote_addr":"[fd00:1122:3344:101::a]:41406","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":18986,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.912703514Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","method":"GET","req_id":"79b601ce-19f0-4391-a8cb-25d3a2130cca","remote_addr":"[fd00:1122:3344:101::a]:41406","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.91357871Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.918130962Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.928199002Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"region":"9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","component":"worker"}
{"msg":"region 9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:12.986501127Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:13.057053021Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/9a9b5978-64fa-41b3-8ab5-2c9e381ad5f8","method":"GET","req_id":"8eb47d80-7776-4028-b5fe-c923484ca940","remote_addr":"[fd00:1122:3344:101::a]:41406","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.239789998Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35551"}
{"msg":"region eb761b5a-4ce8-4d23-9ef4-660d0c7280b7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.299948086Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.300122919Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions","method":"POST","req_id":"9780c6a5-0582-4d78-872a-3719cd48fd44","remote_addr":"[fd00:1122:3344:101::a]:35551","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":764,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.30017183Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.309299714Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.309326994Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"eb761b5a-4ce8-4d23-9ef4-660d0c7280b7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.406265051Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"region":"eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.44021044Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions","method":"POST","req_id":"dbe38258-425b-45a1-94d4-991567a3c38a","remote_addr":"[fd00:1122:3344:101::a]:35551","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":731,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.48763603Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"region":"eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","component":"worker"}
{"msg":"region eb761b5a-4ce8-4d23-9ef4-660d0c7280b7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.48767174Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.488491545Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.490042773Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.49324384Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.503039866Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.50664556Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.507005937Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.507059498Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.507116879Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.50716142Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.5110901Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.516968585Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:04:14.969408254Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions","method":"POST","req_id":"e163a956-87e9-49d8-8b1d-ae73a9bd273b","remote_addr":"[fd00:1122:3344:101::a]:35551","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.993534483Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60263"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.993688426Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","method":"GET","req_id":"95ad28f5-ca71-4ee8-a689-dddc78043a1c","remote_addr":"[fd00:1122:3344:101::a]:60263","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"path is \"/data/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:02.994289017Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.002871691Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"snapshot is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7@fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.010577199Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7\", \"fe1e7c32-a3e0-469a-9e5c-10034bec5870\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.01064279Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.024332626Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7/snapshots/fe1e7c32-a3e0-469a-9e5c-10034bec5870","method":"GET","req_id":"97c9b505-547f-497e-b557-c7f1d1a12ad0","remote_addr":"[fd00:1122:3344:101::a]:60263","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":30593,"response_code":"200"}
{"msg":"path is \"/data/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.024469098Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.032612714Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"snapshot is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7@fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.041606085Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7\", \"fe1e7c32-a3e0-469a-9e5c-10034bec5870\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.041636776Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot eb761b5a-4ce8-4d23-9ef4-660d0c7280b7-fe1e7c32-a3e0-469a-9e5c-10034bec5870 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.051799398Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"applying SMF actions for region eb761b5a-4ce8-4d23-9ef4-660d0c7280b7 running snapshot fe1e7c32-a3e0-469a-9e5c-10034bec5870 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.052352258Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.052390238Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7/snapshots/fe1e7c32-a3e0-469a-9e5c-10034bec5870/run","method":"POST","req_id":"8f0476da-8172-40c2-af6e-654c53eeb940","remote_addr":"[fd00:1122:3344:101::a]:60263","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27954,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-eb761b5a-4ce8-4d23-9ef4-660d0c7280b7-fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.061335989Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-eb761b5a-4ce8-4d23-9ef4-660d0c7280b7-fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.067121262Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-eb761b5a-4ce8-4d23-9ef4-660d0c7280b7-fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.079038976Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eb761b5a-4ce8-4d23-9ef4-660d0c7280b7/.zfs/snapshot/fe1e7c32-a3e0-469a-9e5c-10034bec5870","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.079355492Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.079408553Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.079450753Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.079490324Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.079541025Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.082853454Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.086805205Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"region eb761b5a-4ce8-4d23-9ef4-660d0c7280b7 running snapshot fe1e7c32-a3e0-469a-9e5c-10034bec5870 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:03.086815905Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.890016484Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34538"}
{"msg":"region 2f09d0c6-5b00-4de3-bab8-5b0396c12b29 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.919984561Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.920156894Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions","method":"POST","req_id":"a1dfef84-7e51-460c-aab4-5555c3d73335","remote_addr":"[fd00:1122:3344:101::a]:34538","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":804,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.920179284Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.929558482Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.929603873Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"2f09d0c6-5b00-4de3-bab8-5b0396c12b29\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.996047374Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"region":"2f09d0c6-5b00-4de3-bab8-5b0396c12b29","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.079288386Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"region":"2f09d0c6-5b00-4de3-bab8-5b0396c12b29","component":"worker"}
{"msg":"region 2f09d0c6-5b00-4de3-bab8-5b0396c12b29 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.079349957Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.079467409Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-2f09d0c6-5b00-4de3-bab8-5b0396c12b29","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.080501447Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-2f09d0c6-5b00-4de3-bab8-5b0396c12b29","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.083112844Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.08955217Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-2f09d0c6-5b00-4de3-bab8-5b0396c12b29","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.092594514Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.09290312Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.092970561Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.093035462Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.093099223Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.097308119Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.107064614Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.110604917Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions","method":"POST","req_id":"7c89bd46-ab50-4150-ab6c-fda3291e3983","remote_addr":"[fd00:1122:3344:101::a]:34538","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.787190045Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53093"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.787253626Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29","method":"GET","req_id":"db75eb10-a2c5-461a-a14d-8c22c2cad6e9","remote_addr":"[fd00:1122:3344:101::a]:53093","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"path is \"/data/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.787403928Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.79866391Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"region 2f09d0c6-5b00-4de3-bab8-5b0396c12b29 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.807463958Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.807669561Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.807719492Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29","method":"DELETE","req_id":"5b81b03d-f9f9-49ab-939f-bcef57e7b313","remote_addr":"[fd00:1122:3344:101::a]:53093","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20349,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.808483096Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29","method":"GET","req_id":"08b211fc-56f8-474b-a836-721af0b78955","remote_addr":"[fd00:1122:3344:101::a]:53093","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":635,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-2f09d0c6-5b00-4de3-bab8-5b0396c12b29 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.809587156Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.815651364Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.82432683Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"region":"2f09d0c6-5b00-4de3-bab8-5b0396c12b29","component":"worker"}
{"msg":"region 2f09d0c6-5b00-4de3-bab8-5b0396c12b29 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.876372203Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.88683252Z","hostname":"oxz_crucible_ad4a5561-dc8d-4083-8f16-119c68e57347","pid":22697,"uri":"/crucible/0/regions/2f09d0c6-5b00-4de3-bab8-5b0396c12b29","method":"GET","req_id":"c4016f97-5e46-4c21-bb41-daf8a9f4fd45","remote_addr":"[fd00:1122:3344:101::a]:53093","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":73,"response_code":"200"}