[ Jul  3 18:25:10 Enabled. ]
[ Jul  3 18:25:10 Rereading configuration. ]
[ Jul  3 18:25:13 Rereading configuration. ]
[ Jul  3 18:25:17 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9574af7b-df4d-4b70-8f98-7aeb2f90bab4/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9574af7b-df4d-4b70-8f98-7aeb2f90bab4/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:17.411445942Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:17.412371563Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:17.412385433Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:17.434296411Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:17.699321272Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:37.994033726Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:52509"}
{"msg":"region a743a736-83b9-4989-ba8d-361018aa79ad state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:37.998397037Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.001535154Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"uri":"/crucible/0/regions","method":"POST","req_id":"7340a589-6ed5-48d0-97dd-91fccaf74e36","remote_addr":"[fd00:1122:3344:101::c]:52509","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":6294,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.002271723Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_9574af7b-df4d-4b70-8f98-7aeb2f90bab4/crucible/regions/a743a736-83b9-4989-ba8d-361018aa79ad","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.014163342Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9574af7b-df4d-4b70-8f98-7aeb2f90bab4/crucible/regions/a743a736-83b9-4989-ba8d-361018aa79ad","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.014252003Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a743a736-83b9-4989-ba8d-361018aa79ad\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a743a736-83b9-4989-ba8d-361018aa79ad\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.127083848Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"region":"a743a736-83b9-4989-ba8d-361018aa79ad","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a743a736-83b9-4989-ba8d-361018aa79ad\" \"--data\" \"/data/regions/a743a736-83b9-4989-ba8d-361018aa79ad\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.127177149Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"region":"a743a736-83b9-4989-ba8d-361018aa79ad","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.277627966Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"region":"a743a736-83b9-4989-ba8d-361018aa79ad","component":"worker"}
{"msg":"region a743a736-83b9-4989-ba8d-361018aa79ad state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.277719867Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.27799533Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a743a736-83b9-4989-ba8d-361018aa79ad","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.278413415Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a743a736-83b9-4989-ba8d-361018aa79ad","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.283313293Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.289665587Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"uri":"/crucible/0/regions","method":"POST","req_id":"f81763f1-ca73-41af-9f16-a8e89361ef01","remote_addr":"[fd00:1122:3344:101::c]:52509","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.298296759Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a743a736-83b9-4989-ba8d-361018aa79ad","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.301880201Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a743a736-83b9-4989-ba8d-361018aa79ad","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.30270132Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.305162869Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.30522454Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.305269601Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.315332269Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:38.320853394Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.508735232Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"uri":"/crucible/0/regions/a743a736-83b9-4989-ba8d-361018aa79ad","method":"GET","req_id":"24d2859e-0b01-46f8-9a5e-eff70c6fad15","remote_addr":"[fd00:1122:3344:101::c]:52509","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":98,"response_code":"200"}
{"msg":"path is \"/data/regions/a743a736-83b9-4989-ba8d-361018aa79ad\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.5127787Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"datafile"}
{"msg":"dataset is oxp_9574af7b-df4d-4b70-8f98-7aeb2f90bab4/crucible/regions/a743a736-83b9-4989-ba8d-361018aa79ad","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.523389585Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"datafile"}
{"msg":"region a743a736-83b9-4989-ba8d-361018aa79ad state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.532569302Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.532709514Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.532785975Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"uri":"/crucible/0/regions/a743a736-83b9-4989-ba8d-361018aa79ad","method":"DELETE","req_id":"a6262552-ed03-4ef0-af24-0732007c509c","remote_addr":"[fd00:1122:3344:101::c]:52509","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20076,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-a743a736-83b9-4989-ba8d-361018aa79ad (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.533805907Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.536368787Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"uri":"/crucible/0/regions/a743a736-83b9-4989-ba8d-361018aa79ad","method":"GET","req_id":"ea19fbd4-c87d-4073-a30a-9845a23588df","remote_addr":"[fd00:1122:3344:101::c]:52509","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":30,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.546319374Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9574af7b-df4d-4b70-8f98-7aeb2f90bab4/crucible/regions/a743a736-83b9-4989-ba8d-361018aa79ad\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.556698976Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"region":"a743a736-83b9-4989-ba8d-361018aa79ad","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.598696519Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"uri":"/crucible/0/regions/a743a736-83b9-4989-ba8d-361018aa79ad","method":"GET","req_id":"78a4d0a3-8627-47f7-9faa-5e06bd1faa62","remote_addr":"[fd00:1122:3344:101::c]:52509","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region a743a736-83b9-4989-ba8d-361018aa79ad state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.622572908Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T18:25:39.6967664Z","hostname":"oxz_crucible_da0be91e-7164-4989-a0e3-438153f9c3c9","pid":23611,"uri":"/crucible/0/regions/a743a736-83b9-4989-ba8d-361018aa79ad","method":"GET","req_id":"6ef253db-d978-48c4-9139-de81f1e1e03d","remote_addr":"[fd00:1122:3344:101::c]:52509","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":637,"response_code":"200"}