{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.119774622Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53309"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.119848553Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"uri":"/crucible/0/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc","method":"GET","req_id":"c3c76c18-e33c-40a3-9557-c9e4e099f3a6","remote_addr":"[fd00:1122:3344:101::a]:53309","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"path is \"/data/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.120014355Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"dataset is oxp_553fca36-adc1-4511-ade0-15a4f26d9fb9/crucible/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.129066205Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"snapshot is oxp_553fca36-adc1-4511-ade0-15a4f26d9fb9/crucible/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc@628f48d4-8338-49f7-9dfa-d80baafa49fb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.13784894Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_553fca36-adc1-4511-ade0-15a4f26d9fb9/crucible/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc\", \"628f48d4-8338-49f7-9dfa-d80baafa49fb\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.137901341Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.148665037Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"uri":"/crucible/0/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc/snapshots/628f48d4-8338-49f7-9dfa-d80baafa49fb","method":"GET","req_id":"3be3a98e-09df-466d-ae8e-bb620541f458","remote_addr":"[fd00:1122:3344:101::a]:53309","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":28656,"response_code":"200"}
{"msg":"path is \"/data/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.14884926Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"dataset is oxp_553fca36-adc1-4511-ade0-15a4f26d9fb9/crucible/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.157753597Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"snapshot is oxp_553fca36-adc1-4511-ade0-15a4f26d9fb9/crucible/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc@628f48d4-8338-49f7-9dfa-d80baafa49fb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.166108916Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_553fca36-adc1-4511-ade0-15a4f26d9fb9/crucible/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc\", \"628f48d4-8338-49f7-9dfa-d80baafa49fb\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.166137246Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot ac821bfb-4540-40de-8b2f-9b64bfca3acc-628f48d4-8338-49f7-9dfa-d80baafa49fb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.182252825Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"applying SMF actions for region ac821bfb-4540-40de-8b2f-9b64bfca3acc running snapshot 628f48d4-8338-49f7-9dfa-d80baafa49fb (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.183068117Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.183129678Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"uri":"/crucible/0/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc/snapshots/628f48d4-8338-49f7-9dfa-d80baafa49fb/run","method":"POST","req_id":"3b3318fd-2d2f-4442-8c4e-16bf30902d0d","remote_addr":"[fd00:1122:3344:101::a]:53309","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":34319,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-ac821bfb-4540-40de-8b2f-9b64bfca3acc-628f48d4-8338-49f7-9dfa-d80baafa49fb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.189657489Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-ac821bfb-4540-40de-8b2f-9b64bfca3acc-628f48d4-8338-49f7-9dfa-d80baafa49fb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.201366019Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-ac821bfb-4540-40de-8b2f-9b64bfca3acc-628f48d4-8338-49f7-9dfa-d80baafa49fb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.226463586Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ac821bfb-4540-40de-8b2f-9b64bfca3acc/.zfs/snapshot/628f48d4-8338-49f7-9dfa-d80baafa49fb","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.226756801Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.226815422Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.226856942Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.226897003Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.226939734Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.233620037Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.240919069Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"region ac821bfb-4540-40de-8b2f-9b64bfca3acc running snapshot 628f48d4-8338-49f7-9dfa-d80baafa49fb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:25.240930489Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.282317013Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57672"}
{"msg":"region 68ccb32f-9e94-4750-97c9-78cd5a2ba9df state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.282424924Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.282582347Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.282591907Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"uri":"/crucible/0/regions","method":"POST","req_id":"a8cdc806-f4fd-4e49-a9a9-c371ed8354cb","remote_addr":"[fd00:1122:3344:101::a]:57672","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":208,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_553fca36-adc1-4511-ade0-15a4f26d9fb9/crucible/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.292942327Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_553fca36-adc1-4511-ade0-15a4f26d9fb9/crucible/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.292977347Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"68ccb32f-9e94-4750-97c9-78cd5a2ba9df\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.369339104Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"region":"68ccb32f-9e94-4750-97c9-78cd5a2ba9df","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"68ccb32f-9e94-4750-97c9-78cd5a2ba9df\" \"--data\" \"/data/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.369388465Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"region":"68ccb32f-9e94-4750-97c9-78cd5a2ba9df","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.519008262Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"region":"68ccb32f-9e94-4750-97c9-78cd5a2ba9df","component":"worker"}
{"msg":"region 68ccb32f-9e94-4750-97c9-78cd5a2ba9df state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.519047032Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.519150484Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-68ccb32f-9e94-4750-97c9-78cd5a2ba9df","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.52018315Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-68ccb32f-9e94-4750-97c9-78cd5a2ba9df","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.528574909Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.543198555Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"uri":"/crucible/0/regions","method":"POST","req_id":"0e4da1e2-3b05-47c9-bfa3-2d6c646c2886","remote_addr":"[fd00:1122:3344:101::a]:57672","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.544084588Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-68ccb32f-9e94-4750-97c9-78cd5a2ba9df","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.54813066Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.548411675Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.548516826Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.548567507Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.548622688Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.558376318Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:28.576608699Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.485470487Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"uri":"/crucible/0/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df","method":"GET","req_id":"b1c0ea1b-bdd2-4094-b2b4-a1023f35368b","remote_addr":"[fd00:1122:3344:101::a]:57672","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"path is \"/data/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.488774488Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"dataset is oxp_553fca36-adc1-4511-ade0-15a4f26d9fb9/crucible/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.498842983Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"region 68ccb32f-9e94-4750-97c9-78cd5a2ba9df state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.507570677Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.50775017Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.507797821Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"uri":"/crucible/0/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df","method":"DELETE","req_id":"503d623c-757a-44df-bff2-09021fd95f05","remote_addr":"[fd00:1122:3344:101::a]:57672","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19047,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-68ccb32f-9e94-4750-97c9-78cd5a2ba9df (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.509464697Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.510639225Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"uri":"/crucible/0/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df","method":"GET","req_id":"2950c151-f6c8-46ab-bea3-37c4e5cc4666","remote_addr":"[fd00:1122:3344:101::a]:57672","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.514661377Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_553fca36-adc1-4511-ade0-15a4f26d9fb9/crucible/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.523603254Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"region":"68ccb32f-9e94-4750-97c9-78cd5a2ba9df","component":"worker"}
{"msg":"region 68ccb32f-9e94-4750-97c9-78cd5a2ba9df state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.57195732Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T02:15:53.583211804Z","hostname":"oxz_crucible_d275e9ef-209a-4386-8746-e2654671f435","pid":21926,"uri":"/crucible/0/regions/68ccb32f-9e94-4750-97c9-78cd5a2ba9df","method":"GET","req_id":"942a0a0f-50c9-4221-b125-725c81f72e4a","remote_addr":"[fd00:1122:3344:101::a]:57672","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":48,"response_code":"200"}