[ Jul  3 17:22:24 Enabled. ]
[ Jul  3 17:22:24 Rereading configuration. ]
[ Jul  3 17:22:26 Rereading configuration. ]
[ Jul  3 17:22:31 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.48546972Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.490598934Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.490613264Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.505477533Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:22:31.707295026Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.418544724Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63074"}
{"msg":"region dad1bc4e-caa2-435f-924d-78418179ab56 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.427384415Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.433811854Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions","method":"POST","req_id":"ae5d93c6-c914-48b7-bb31-2e8b324efcf6","remote_addr":"[fd00:1122:3344:101::a]:63074","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":12836,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.436064442Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/dad1bc4e-caa2-435f-924d-78418179ab56","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.444626897Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/dad1bc4e-caa2-435f-924d-78418179ab56","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.444658017Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"dad1bc4e-caa2-435f-924d-78418179ab56\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/dad1bc4e-caa2-435f-924d-78418179ab56\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.504940615Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"region":"dad1bc4e-caa2-435f-924d-78418179ab56","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"dad1bc4e-caa2-435f-924d-78418179ab56\" \"--data\" \"/data/regions/dad1bc4e-caa2-435f-924d-78418179ab56\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.507211215Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"region":"dad1bc4e-caa2-435f-924d-78418179ab56","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.715497747Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions","method":"POST","req_id":"3c370683-fae2-4c7a-a383-7256c9f83392","remote_addr":"[fd00:1122:3344:101::a]:63074","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.811701125Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"region":"dad1bc4e-caa2-435f-924d-78418179ab56","component":"worker"}
{"msg":"region dad1bc4e-caa2-435f-924d-78418179ab56 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.811740896Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.81188041Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-dad1bc4e-caa2-435f-924d-78418179ab56","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.812352882Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dad1bc4e-caa2-435f-924d-78418179ab56","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.820738991Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.836405991Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dad1bc4e-caa2-435f-924d-78418179ab56","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.842730517Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dad1bc4e-caa2-435f-924d-78418179ab56","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.843783914Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.843890427Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.843937389Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.843968949Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.851444975Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:43.863929392Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:23:44.264098915Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions","method":"POST","req_id":"65883376-6968-4a0a-8679-49cb533168f2","remote_addr":"[fd00:1122:3344:101::a]:63074","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":90,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.32722889Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49327"}
{"msg":"region 0ac17eed-4e75-4332-b410-6ca9052c0c55 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.327359283Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.328150254Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.328177475Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions","method":"POST","req_id":"8ec11f60-eaba-44c1-b359-1bc3c81d6e0b","remote_addr":"[fd00:1122:3344:101::a]:49327","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":834,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/0ac17eed-4e75-4332-b410-6ca9052c0c55","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.339517552Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/0ac17eed-4e75-4332-b410-6ca9052c0c55","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.339548412Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0ac17eed-4e75-4332-b410-6ca9052c0c55\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0ac17eed-4e75-4332-b410-6ca9052c0c55\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.399657316Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"region":"0ac17eed-4e75-4332-b410-6ca9052c0c55","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0ac17eed-4e75-4332-b410-6ca9052c0c55\" \"--data\" \"/data/regions/0ac17eed-4e75-4332-b410-6ca9052c0c55\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.399711617Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"region":"0ac17eed-4e75-4332-b410-6ca9052c0c55","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.485540544Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions","method":"POST","req_id":"af91afa1-91a6-4982-9c40-b5bc57549d6b","remote_addr":"[fd00:1122:3344:101::a]:49327","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":681,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.497002414Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"region":"0ac17eed-4e75-4332-b410-6ca9052c0c55","component":"worker"}
{"msg":"region 0ac17eed-4e75-4332-b410-6ca9052c0c55 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.497028704Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.497140387Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0ac17eed-4e75-4332-b410-6ca9052c0c55","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.497531958Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0ac17eed-4e75-4332-b410-6ca9052c0c55","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.502536339Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.518915507Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0ac17eed-4e75-4332-b410-6ca9052c0c55","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.521832194Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0ac17eed-4e75-4332-b410-6ca9052c0c55","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.522133212Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.522180203Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.522218554Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.522254585Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.525386457Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:07.534476155Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.010725101Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions","method":"POST","req_id":"19e88d96-8847-42dc-bbf7-60bd057f6dbf","remote_addr":"[fd00:1122:3344:101::a]:49327","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.620712367Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43469"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.620867831Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions/dad1bc4e-caa2-435f-924d-78418179ab56","method":"GET","req_id":"2d5bfa41-2cb3-4ede-aa32-a88c4ba341b6","remote_addr":"[fd00:1122:3344:101::a]:43469","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":89,"response_code":"200"}
{"msg":"path is \"/data/regions/dad1bc4e-caa2-435f-924d-78418179ab56\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.625483152Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"dataset is oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/dad1bc4e-caa2-435f-924d-78418179ab56","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.639454388Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"snapshot is oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/dad1bc4e-caa2-435f-924d-78418179ab56@f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.651821751Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/dad1bc4e-caa2-435f-924d-78418179ab56\", \"f21c53cc-7f60-4c98-85eb-88fe670d1630\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.651864703Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.667597184Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions/dad1bc4e-caa2-435f-924d-78418179ab56/snapshots/f21c53cc-7f60-4c98-85eb-88fe670d1630","method":"GET","req_id":"f613d527-28ef-41fd-8055-e67a84b1b7e7","remote_addr":"[fd00:1122:3344:101::a]:43469","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":46472,"response_code":"200"}
{"msg":"path is \"/data/regions/dad1bc4e-caa2-435f-924d-78418179ab56\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.667973604Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"dataset is oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/dad1bc4e-caa2-435f-924d-78418179ab56","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.680617645Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"snapshot is oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/dad1bc4e-caa2-435f-924d-78418179ab56@f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.692494146Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/dad1bc4e-caa2-435f-924d-78418179ab56\", \"f21c53cc-7f60-4c98-85eb-88fe670d1630\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.692544067Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot dad1bc4e-caa2-435f-924d-78418179ab56-f21c53cc-7f60-4c98-85eb-88fe670d1630 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.705439375Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"applying SMF actions for region dad1bc4e-caa2-435f-924d-78418179ab56 running snapshot f21c53cc-7f60-4c98-85eb-88fe670d1630 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.705695332Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.705716732Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions/dad1bc4e-caa2-435f-924d-78418179ab56/snapshots/f21c53cc-7f60-4c98-85eb-88fe670d1630/run","method":"POST","req_id":"5f9ab526-ade9-4334-a067-609918ecbff9","remote_addr":"[fd00:1122:3344:101::a]:43469","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":37819,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-dad1bc4e-caa2-435f-924d-78418179ab56-f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.70907299Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-dad1bc4e-caa2-435f-924d-78418179ab56-f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.717962303Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-dad1bc4e-caa2-435f-924d-78418179ab56-f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.730120901Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dad1bc4e-caa2-435f-924d-78418179ab56/.zfs/snapshot/f21c53cc-7f60-4c98-85eb-88fe670d1630","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.730395578Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.73046413Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.730522151Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.730580113Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.730641325Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.735010829Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.739682181Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"region dad1bc4e-caa2-435f-924d-78418179ab56 running snapshot f21c53cc-7f60-4c98-85eb-88fe670d1630 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:08.739691651Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"region 7e565daa-43db-424c-b412-571c00cfa39f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.565281991Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.565423265Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions","method":"POST","req_id":"7b26fed3-ed70-4aa4-bf73-2cfd41a80a0d","remote_addr":"[fd00:1122:3344:101::a]:49327","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":208,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.565453576Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/7e565daa-43db-424c-b412-571c00cfa39f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.574141814Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/7e565daa-43db-424c-b412-571c00cfa39f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.574174804Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"7e565daa-43db-424c-b412-571c00cfa39f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7e565daa-43db-424c-b412-571c00cfa39f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.623930557Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"region":"7e565daa-43db-424c-b412-571c00cfa39f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7e565daa-43db-424c-b412-571c00cfa39f\" \"--data\" \"/data/regions/7e565daa-43db-424c-b412-571c00cfa39f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.625830587Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"region":"7e565daa-43db-424c-b412-571c00cfa39f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.692310247Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"region":"7e565daa-43db-424c-b412-571c00cfa39f","component":"worker"}
{"msg":"region 7e565daa-43db-424c-b412-571c00cfa39f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.692357668Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.692478361Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-7e565daa-43db-424c-b412-571c00cfa39f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.693720824Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7e565daa-43db-424c-b412-571c00cfa39f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.69739299Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.705197744Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7e565daa-43db-424c-b412-571c00cfa39f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.708297165Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7e565daa-43db-424c-b412-571c00cfa39f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.708559522Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.708614143Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.708652254Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.708688635Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.712511435Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.723106763Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:11.890626198Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions","method":"POST","req_id":"2cc8c93d-e54a-459d-82c5-a31521c3724e","remote_addr":"[fd00:1122:3344:101::a]:49327","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":590,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.670259558Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions/7e565daa-43db-424c-b412-571c00cfa39f","method":"GET","req_id":"65a10db8-0dd4-4117-9c00-3609037702be","remote_addr":"[fd00:1122:3344:101::a]:49327","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"path is \"/data/regions/7e565daa-43db-424c-b412-571c00cfa39f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.673262007Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"dataset is oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/7e565daa-43db-424c-b412-571c00cfa39f","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.683027872Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"region 7e565daa-43db-424c-b412-571c00cfa39f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.691284818Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.691531495Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.691608407Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions/7e565daa-43db-424c-b412-571c00cfa39f","method":"DELETE","req_id":"80d3d58a-1071-44da-9587-8c5b2837bc31","remote_addr":"[fd00:1122:3344:101::a]:49327","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18899,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-7e565daa-43db-424c-b412-571c00cfa39f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.69286341Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.694328378Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions/7e565daa-43db-424c-b412-571c00cfa39f","method":"GET","req_id":"1f294bb9-a161-4cd8-abe9-813e5513fb16","remote_addr":"[fd00:1122:3344:101::a]:49327","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.699210336Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_da517756-cb4c-4c7d-801d-76be4f2c6b99/crucible/regions/7e565daa-43db-424c-b412-571c00cfa39f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.709086084Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"region":"7e565daa-43db-424c-b412-571c00cfa39f","component":"worker"}
{"msg":"region 7e565daa-43db-424c-b412-571c00cfa39f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.753951829Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-03T17:29:36.785944776Z","hostname":"oxz_crucible_c2a9ff1b-6bf2-44ab-820a-ffa0bdc5e4f1","pid":26620,"uri":"/crucible/0/regions/7e565daa-43db-424c-b412-571c00cfa39f","method":"GET","req_id":"9858b5f0-850d-4e91-87da-1a0fd88a07a1","remote_addr":"[fd00:1122:3344:101::a]:49327","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":505,"response_code":"200"}