[ May 30 04:27:47 Enabled. ]
[ May 30 04:27:47 Rereading configuration. ]
[ May 30 04:27:49 Rereading configuration. ]
[ May 30 04:27:52 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_677db61e-a49b-4c52-b7ee-92a1f2d4ec52/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_677db61e-a49b-4c52-b7ee-92a1f2d4ec52/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:27:52.102648942Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:27:52.106060351Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:27:52.106083651Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:27:52.121857273Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:27:52.306230744Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.653657648Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:55370"}
{"msg":"region 63c9fca0-80dc-4687-8350-f4c7ee819e2a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.714561849Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.717821677Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.722353138Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"uri":"/crucible/0/regions","method":"POST","req_id":"230703dc-94c3-45ba-8644-9dd747ad0bd7","remote_addr":"[fd00:1122:3344:101::c]:55370","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":11992,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_677db61e-a49b-4c52-b7ee-92a1f2d4ec52/crucible/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.728926714Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_677db61e-a49b-4c52-b7ee-92a1f2d4ec52/crucible/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.728984955Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"63c9fca0-80dc-4687-8350-f4c7ee819e2a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.804336282Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"region":"63c9fca0-80dc-4687-8350-f4c7ee819e2a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"63c9fca0-80dc-4687-8350-f4c7ee819e2a\" \"--data\" \"/data/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.806214263Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"region":"63c9fca0-80dc-4687-8350-f4c7ee819e2a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.934091775Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"region":"63c9fca0-80dc-4687-8350-f4c7ee819e2a","component":"worker"}
{"msg":"region 63c9fca0-80dc-4687-8350-f4c7ee819e2a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.934183466Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.934366858Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-63c9fca0-80dc-4687-8350-f4c7ee819e2a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.934763953Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-63c9fca0-80dc-4687-8350-f4c7ee819e2a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.939956552Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.947920194Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-63c9fca0-80dc-4687-8350-f4c7ee819e2a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.952703059Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.958161872Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.960283876Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.960353997Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.960420248Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.965333464Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:30.969598353Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.028793194Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"uri":"/crucible/0/regions","method":"POST","req_id":"57241309-5d63-43e2-afc5-21e93e4d3b01","remote_addr":"[fd00:1122:3344:101::c]:55370","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.671818762Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:56796"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.671951143Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"uri":"/crucible/0/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a","method":"GET","req_id":"4e7e6b4a-d6c1-40a9-9c5d-05c9727bf355","remote_addr":"[fd00:1122:3344:101::c]:56796","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"path is \"/data/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.672252427Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"datafile"}
{"msg":"dataset is oxp_677db61e-a49b-4c52-b7ee-92a1f2d4ec52/crucible/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.681996609Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"datafile"}
{"msg":"region 63c9fca0-80dc-4687-8350-f4c7ee819e2a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.690483436Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.690698049Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-63c9fca0-80dc-4687-8350-f4c7ee819e2a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.692274477Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.693089806Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"uri":"/crucible/0/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a","method":"DELETE","req_id":"19bda3b7-19aa-4907-aae8-af4303fcb9c9","remote_addr":"[fd00:1122:3344:101::c]:56796","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20880,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.693325009Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"uri":"/crucible/0/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a","method":"GET","req_id":"43bdd41b-b68f-439a-bac8-6036280a4cee","remote_addr":"[fd00:1122:3344:101::c]:56796","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":21,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.696230683Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_677db61e-a49b-4c52-b7ee-92a1f2d4ec52/crucible/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.706739454Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"region":"63c9fca0-80dc-4687-8350-f4c7ee819e2a","component":"worker"}
{"msg":"region 63c9fca0-80dc-4687-8350-f4c7ee819e2a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.76550346Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:28:31.820646794Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"uri":"/crucible/0/regions/63c9fca0-80dc-4687-8350-f4c7ee819e2a","method":"GET","req_id":"d568dabb-f6c5-4e9b-8207-566e796313e2","remote_addr":"[fd00:1122:3344:101::c]:56796","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.028990582Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40623"}
{"msg":"region e23dc52e-513c-4b17-a9c9-d6ff6816825c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.088302874Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.088479886Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"uri":"/crucible/0/regions","method":"POST","req_id":"6cb1d606-1b99-4b07-aae6-d0194fdf0777","remote_addr":"[fd00:1122:3344:101::a]:40623","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":235,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.088499766Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_677db61e-a49b-4c52-b7ee-92a1f2d4ec52/crucible/regions/e23dc52e-513c-4b17-a9c9-d6ff6816825c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.096712741Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_677db61e-a49b-4c52-b7ee-92a1f2d4ec52/crucible/regions/e23dc52e-513c-4b17-a9c9-d6ff6816825c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.096750161Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e23dc52e-513c-4b17-a9c9-d6ff6816825c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e23dc52e-513c-4b17-a9c9-d6ff6816825c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.145993428Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"region":"e23dc52e-513c-4b17-a9c9-d6ff6816825c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e23dc52e-513c-4b17-a9c9-d6ff6816825c\" \"--data\" \"/data/regions/e23dc52e-513c-4b17-a9c9-d6ff6816825c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.14793204Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"region":"e23dc52e-513c-4b17-a9c9-d6ff6816825c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.20438483Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"region":"e23dc52e-513c-4b17-a9c9-d6ff6816825c","component":"worker"}
{"msg":"region e23dc52e-513c-4b17-a9c9-d6ff6816825c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.2044196Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.204527892Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e23dc52e-513c-4b17-a9c9-d6ff6816825c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.205917417Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e23dc52e-513c-4b17-a9c9-d6ff6816825c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.208602908Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.215013362Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e23dc52e-513c-4b17-a9c9-d6ff6816825c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.217681693Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e23dc52e-513c-4b17-a9c9-d6ff6816825c","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.217945016Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.218005696Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.218047597Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.218089657Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.22176387Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.225850147Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-30T04:29:04.377727304Z","hostname":"oxz_crucible_08d23232-5878-435d-88c7-c09d6134f50a","pid":22904,"uri":"/crucible/0/regions","method":"POST","req_id":"65921ffe-e0c6-43a5-970d-b42e5160e629","remote_addr":"[fd00:1122:3344:101::a]:40623","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":79,"response_code":"200"}