{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.515569274Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64281"} {"msg":"region 12e68e5b-0461-4b91-90f4-f1af578f0b1c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.547705207Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.552692331Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions","method":"POST","req_id":"46f0cd60-6f1f-4056-8e4a-938ea1f317c7","remote_addr":"[fd00:1122:3344:101::a]:64281","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":22752,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.556594289Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.56865453Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.56867608Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"12e68e5b-0461-4b91-90f4-f1af578f0b1c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.614011243Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"region":"12e68e5b-0461-4b91-90f4-f1af578f0b1c","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"12e68e5b-0461-4b91-90f4-f1af578f0b1c\" \"--data\" \"/data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.617883529Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"region":"12e68e5b-0461-4b91-90f4-f1af578f0b1c","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.859903452Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"region":"12e68e5b-0461-4b91-90f4-f1af578f0b1c","component":"worker"} {"msg":"region 12e68e5b-0461-4b91-90f4-f1af578f0b1c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.859961813Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.860047455Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.862395594Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.867348768Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.88467225Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.887931892Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.896026494Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions","method":"POST","req_id":"f426f700-02ce-48db-b49f-6c15e9ba051d","remote_addr":"[fd00:1122:3344:101::a]:64281","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":70,"response_code":"200"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.898116886Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.900142696Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.900208998Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.90025889Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.904498405Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:31:25.910544766Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.489306894Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33152"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.489395576Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","method":"GET","req_id":"ec4a3102-d6f9-4d36-95f9-cd233e4f086b","remote_addr":"[fd00:1122:3344:101::a]:33152","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":48,"response_code":"200"} {"msg":"path is \"/data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.493606361Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"dataset is oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.502231317Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"snapshot is oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c@45c10264-cac8-4da8-85cc-3ec557aec3bd","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.510118724Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c\", \"45c10264-cac8-4da8-85cc-3ec557aec3bd\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.510157435Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.522706698Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c/snapshots/45c10264-cac8-4da8-85cc-3ec557aec3bd","method":"GET","req_id":"6c6956c8-9f50-4b49-ac78-afb73c641c50","remote_addr":"[fd00:1122:3344:101::a]:33152","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":29688,"response_code":"200"} {"msg":"path is \"/data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.526419181Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"dataset is oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.534165364Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"snapshot is oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c@45c10264-cac8-4da8-85cc-3ec557aec3bd","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.541686012Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c\", \"45c10264-cac8-4da8-85cc-3ec557aec3bd\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.541720373Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 12e68e5b-0461-4b91-90f4-f1af578f0b1c-45c10264-cac8-4da8-85cc-3ec557aec3bd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.549736783Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"applying SMF actions for region 12e68e5b-0461-4b91-90f4-f1af578f0b1c running snapshot 45c10264-cac8-4da8-85cc-3ec557aec3bd (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.550382059Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.550442551Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c/snapshots/45c10264-cac8-4da8-85cc-3ec557aec3bd/run","method":"POST","req_id":"ada0523c-b4f1-4815-8349-3c3a0ede2ed7","remote_addr":"[fd00:1122:3344:101::a]:33152","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24056,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.56121786Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.561306202Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.561386874Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.561394284Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.561434825Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.561482926Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.561580499Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.563874766Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.563966198Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.564058841Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.571759993Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-12e68e5b-0461-4b91-90f4-f1af578f0b1c-45c10264-cac8-4da8-85cc-3ec557aec3bd","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.572332347Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-12e68e5b-0461-4b91-90f4-f1af578f0b1c-45c10264-cac8-4da8-85cc-3ec557aec3bd","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.580640155Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-12e68e5b-0461-4b91-90f4-f1af578f0b1c-45c10264-cac8-4da8-85cc-3ec557aec3bd","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.598987853Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c/.zfs/snapshot/45c10264-cac8-4da8-85cc-3ec557aec3bd","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.599241269Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.599310031Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.599353252Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.599393593Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.599453075Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.60610061Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.612966862Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"region 12e68e5b-0461-4b91-90f4-f1af578f0b1c running snapshot 45c10264-cac8-4da8-85cc-3ec557aec3bd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.612993513Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"path is \"/data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.915402513Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"dataset is oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.924584562Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"snapshot is oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c@45c10264-cac8-4da8-85cc-3ec557aec3bd","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.932255834Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c\", \"45c10264-cac8-4da8-85cc-3ec557aec3bd\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.932289895Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:41.939538656Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c/snapshots/45c10264-cac8-4da8-85cc-3ec557aec3bd/run","method":"POST","req_id":"d68ae902-5289-480d-a276-1e9c1a900b31","remote_addr":"[fd00:1122:3344:101::a]:33152","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24184,"response_code":"200"} {"msg":"region de91351c-9f1d-4aa8-8af2-709b0bb49a8f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.386108956Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.3862547Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions","method":"POST","req_id":"e43ac412-6b5e-4a84-9b7b-c1e7d02f4d87","remote_addr":"[fd00:1122:3344:101::a]:33152","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":212,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.386291841Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.395976823Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.396006144Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"de91351c-9f1d-4aa8-8af2-709b0bb49a8f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.453235933Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"region":"de91351c-9f1d-4aa8-8af2-709b0bb49a8f","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"de91351c-9f1d-4aa8-8af2-709b0bb49a8f\" \"--data\" \"/data/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.453293594Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"region":"de91351c-9f1d-4aa8-8af2-709b0bb49a8f","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.533866496Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"region":"de91351c-9f1d-4aa8-8af2-709b0bb49a8f","component":"worker"} {"msg":"region de91351c-9f1d-4aa8-8af2-709b0bb49a8f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.533927388Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.534052781Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.534736458Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.53481361Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.534912952Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.534920652Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.534965314Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.535015105Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.535400664Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.535463296Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.535517907Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.535563628Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.540000259Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-de91351c-9f1d-4aa8-8af2-709b0bb49a8f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.540454401Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-de91351c-9f1d-4aa8-8af2-709b0bb49a8f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.547921467Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.554701086Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-de91351c-9f1d-4aa8-8af2-709b0bb49a8f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.557716992Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.557915977Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.557975478Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.558016999Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.5580553Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.561681281Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.56724185Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:36:45.726473275Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions","method":"POST","req_id":"c65326e0-ab51-45aa-9a4b-8e7a510ecdbe","remote_addr":"[fd00:1122:3344:101::a]:33152","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":69,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.946576459Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f","method":"GET","req_id":"b08b4b26-4971-44bd-bc5d-4548a35b3749","remote_addr":"[fd00:1122:3344:101::a]:33152","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":49,"response_code":"200"} {"msg":"path is \"/data/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.949044411Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"dataset is oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.959681136Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"region de91351c-9f1d-4aa8-8af2-709b0bb49a8f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.967788699Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.967987524Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.968052015Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f","method":"DELETE","req_id":"6fd41798-898b-42a1-8cfd-f80e2c8e447f","remote_addr":"[fd00:1122:3344:101::a]:33152","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19037,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-de91351c-9f1d-4aa8-8af2-709b0bb49a8f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.969283646Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.970840265Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f","method":"GET","req_id":"eb2ef9df-29b7-4aaa-8017-a32e84a51e51","remote_addr":"[fd00:1122:3344:101::a]:33152","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.97385652Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.973954793Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.974038635Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.974051825Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.974088056Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.974135297Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/12e68e5b-0461-4b91-90f4-f1af578f0b1c","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.974481666Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.974544527Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.974604559Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.97466633Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.978692971Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.979990273Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_f936d8c3-1ac2-4fd5-84ed-415a39983d55/crucible/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:10.989535382Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"region":"de91351c-9f1d-4aa8-8af2-709b0bb49a8f","component":"worker"} {"msg":"region de91351c-9f1d-4aa8-8af2-709b0bb49a8f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:11.037211722Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T21:37:11.046384951Z","hostname":"oxz_crucible_bab56a2f-758f-46b5-a644-c8903658005a","pid":20910,"uri":"/crucible/0/regions/de91351c-9f1d-4aa8-8af2-709b0bb49a8f","method":"GET","req_id":"149a9e65-6eb7-4d3b-ae2a-d41f578eafe5","remote_addr":"[fd00:1122:3344:101::a]:33152","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":65,"response_code":"200"}