[ Jun 25 04:50:02 Enabled. ]
[ Jun 25 04:50:02 Rereading configuration. ]
[ Jun 25 04:50:04 Rereading configuration. ]
[ Jun 25 04:50:07 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_06c5d615-3101-4280-b654-568002756968/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_06c5d615-3101-4280-b654-568002756968/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:07.076757584Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:07.078257915Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:07.078280126Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:07.099522797Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:07.203163212Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.934170558Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42060"}
{"msg":"region f0233c31-e9d9-4ed3-87d2-d6f661232050 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.942296247Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.943377725Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions","method":"POST","req_id":"ec7c4509-00c5-48de-ac2c-e2b39ee28d74","remote_addr":"[fd00:1122:3344:101::a]:42060","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":1478,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.943394775Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.953052149Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:25.95308435Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f0233c31-e9d9-4ed3-87d2-d6f661232050\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.014330638Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"f0233c31-e9d9-4ed3-87d2-d6f661232050","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f0233c31-e9d9-4ed3-87d2-d6f661232050\" \"--data\" \"/data/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.014406249Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"f0233c31-e9d9-4ed3-87d2-d6f661232050","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.137348612Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"f0233c31-e9d9-4ed3-87d2-d6f661232050","component":"worker"}
{"msg":"region f0233c31-e9d9-4ed3-87d2-d6f661232050 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.137401933Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.137478204Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f0233c31-e9d9-4ed3-87d2-d6f661232050","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.13789254Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f0233c31-e9d9-4ed3-87d2-d6f661232050","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.141216834Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.15226157Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f0233c31-e9d9-4ed3-87d2-d6f661232050","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.157142098Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.161876614Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.163845615Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.163921606Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.163973077Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.168788154Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.173090302Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.228843723Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions","method":"POST","req_id":"f2c6e918-5ae8-4f54-b2d2-90dbd5d162ec","remote_addr":"[fd00:1122:3344:101::a]:42060","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.635168471Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050","method":"GET","req_id":"9cd007d0-41fb-44f1-a8b9-9bac55c38ac9","remote_addr":"[fd00:1122:3344:101::a]:42060","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"path is \"/data/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.637125982Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"dataset is oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.646662214Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"region f0233c31-e9d9-4ed3-87d2-d6f661232050 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.656300998Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.656459761Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.656570513Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050","method":"DELETE","req_id":"d3593bae-1ef7-433a-905e-e50f04e9c80f","remote_addr":"[fd00:1122:3344:101::a]:42060","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":19488,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-f0233c31-e9d9-4ed3-87d2-d6f661232050 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.657623309Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.659420938Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050","method":"GET","req_id":"2876f651-ea26-4b62-8df7-9f13774bf70f","remote_addr":"[fd00:1122:3344:101::a]:42060","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.667892513Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.677275503Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"f0233c31-e9d9-4ed3-87d2-d6f661232050","component":"worker"}
{"msg":"region f0233c31-e9d9-4ed3-87d2-d6f661232050 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.741768133Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:26.77225056Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/f0233c31-e9d9-4ed3-87d2-d6f661232050","method":"GET","req_id":"3da2beae-5a4e-4fad-a17d-fc2c5e5d28c1","remote_addr":"[fd00:1122:3344:101::a]:42060","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"region 51090670-87a5-419d-8293-a431f9db6ab7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.213814537Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.213948839Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions","method":"POST","req_id":"dc6735f7-43ea-4364-8d5b-c882aa2a7c81","remote_addr":"[fd00:1122:3344:101::a]:42060","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":205,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.2139664Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/51090670-87a5-419d-8293-a431f9db6ab7","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.225131548Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/51090670-87a5-419d-8293-a431f9db6ab7","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.225175639Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"51090670-87a5-419d-8293-a431f9db6ab7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/51090670-87a5-419d-8293-a431f9db6ab7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.303487379Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"51090670-87a5-419d-8293-a431f9db6ab7","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"51090670-87a5-419d-8293-a431f9db6ab7\" \"--data\" \"/data/regions/51090670-87a5-419d-8293-a431f9db6ab7\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.305974869Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"51090670-87a5-419d-8293-a431f9db6ab7","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.373260193Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"51090670-87a5-419d-8293-a431f9db6ab7","component":"worker"}
{"msg":"region 51090670-87a5-419d-8293-a431f9db6ab7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.373318484Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.373410645Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-51090670-87a5-419d-8293-a431f9db6ab7","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.374904549Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-51090670-87a5-419d-8293-a431f9db6ab7","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.378037325Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.385932225Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-51090670-87a5-419d-8293-a431f9db6ab7","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.388692699Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/51090670-87a5-419d-8293-a431f9db6ab7","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.388922713Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.388980224Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.389024345Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.389063445Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.392131554Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.395995236Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:50:28.4231674Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions","method":"POST","req_id":"5904614d-e028-43cd-a33c-17ca16ee22d5","remote_addr":"[fd00:1122:3344:101::a]:42060","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.773829265Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40514"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.774006088Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/51090670-87a5-419d-8293-a431f9db6ab7","method":"GET","req_id":"66157b0a-b746-40ed-9b70-b9629e763f85","remote_addr":"[fd00:1122:3344:101::a]:40514","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"path is \"/data/regions/51090670-87a5-419d-8293-a431f9db6ab7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.776834533Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"dataset is oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/51090670-87a5-419d-8293-a431f9db6ab7","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.786960114Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"snapshot is oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/51090670-87a5-419d-8293-a431f9db6ab7@d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.79669543Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/51090670-87a5-419d-8293-a431f9db6ab7\", \"d7341147-ce54-48ba-ab4a-2579a151f90d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.79672764Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.811233642Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/51090670-87a5-419d-8293-a431f9db6ab7/snapshots/d7341147-ce54-48ba-ab4a-2579a151f90d","method":"GET","req_id":"f3a9ae4e-12ec-490e-89fa-36630fdec141","remote_addr":"[fd00:1122:3344:101::a]:40514","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":37171,"response_code":"200"}
{"msg":"path is \"/data/regions/51090670-87a5-419d-8293-a431f9db6ab7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.811435015Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"dataset is oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/51090670-87a5-419d-8293-a431f9db6ab7","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.820420919Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"snapshot is oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/51090670-87a5-419d-8293-a431f9db6ab7@d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.829486214Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/51090670-87a5-419d-8293-a431f9db6ab7\", \"d7341147-ce54-48ba-ab4a-2579a151f90d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.829515754Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 51090670-87a5-419d-8293-a431f9db6ab7-d7341147-ce54-48ba-ab4a-2579a151f90d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.838328495Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"applying SMF actions for region 51090670-87a5-419d-8293-a431f9db6ab7 running snapshot d7341147-ce54-48ba-ab4a-2579a151f90d (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.838927664Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.838976455Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/51090670-87a5-419d-8293-a431f9db6ab7/snapshots/d7341147-ce54-48ba-ab4a-2579a151f90d/run","method":"POST","req_id":"220853c2-7d69-4471-9164-fbe03d4ec994","remote_addr":"[fd00:1122:3344:101::a]:40514","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":27588,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-51090670-87a5-419d-8293-a431f9db6ab7-d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.843974905Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-51090670-87a5-419d-8293-a431f9db6ab7-d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.848508237Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-51090670-87a5-419d-8293-a431f9db6ab7-d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.859299359Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/51090670-87a5-419d-8293-a431f9db6ab7/.zfs/snapshot/d7341147-ce54-48ba-ab4a-2579a151f90d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.859496973Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.859550183Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.859592114Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.859633965Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.859693206Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.862913697Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.867032663Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"region 51090670-87a5-419d-8293-a431f9db6ab7 running snapshot d7341147-ce54-48ba-ab4a-2579a151f90d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:42.867052463Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.504956333Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63168"}
{"msg":"region 4859e6da-cd85-4148-ad35-1841edefa459 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.505127076Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.505276248Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions","method":"POST","req_id":"368f262a-2ba1-4171-b11c-8f39ce365f4d","remote_addr":"[fd00:1122:3344:101::a]:63168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":200,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.505294189Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/4859e6da-cd85-4148-ad35-1841edefa459","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.515058765Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/4859e6da-cd85-4148-ad35-1841edefa459","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.515092615Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4859e6da-cd85-4148-ad35-1841edefa459\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4859e6da-cd85-4148-ad35-1841edefa459\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.573956495Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"4859e6da-cd85-4148-ad35-1841edefa459","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4859e6da-cd85-4148-ad35-1841edefa459\" \"--data\" \"/data/regions/4859e6da-cd85-4148-ad35-1841edefa459\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.576027957Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"4859e6da-cd85-4148-ad35-1841edefa459","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.638641947Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"4859e6da-cd85-4148-ad35-1841edefa459","component":"worker"}
{"msg":"region 4859e6da-cd85-4148-ad35-1841edefa459 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.638682238Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.63878237Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4859e6da-cd85-4148-ad35-1841edefa459","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.639787515Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4859e6da-cd85-4148-ad35-1841edefa459","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.642468018Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.649547471Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4859e6da-cd85-4148-ad35-1841edefa459","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.652859064Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4859e6da-cd85-4148-ad35-1841edefa459","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.653066608Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.653133559Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.653175839Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.65321578Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.656133587Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.664951507Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:55:45.769890933Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions","method":"POST","req_id":"ba48efe3-ede4-4e30-aed9-0f803ea0ed20","remote_addr":"[fd00:1122:3344:101::a]:63168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.428343131Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/4859e6da-cd85-4148-ad35-1841edefa459","method":"GET","req_id":"5c7c1d9f-b092-4d07-bc84-c93816bcf22c","remote_addr":"[fd00:1122:3344:101::a]:63168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":66,"response_code":"200"}
{"msg":"path is \"/data/regions/4859e6da-cd85-4148-ad35-1841edefa459\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.431319719Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"dataset is oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/4859e6da-cd85-4148-ad35-1841edefa459","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.441864587Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"region 4859e6da-cd85-4148-ad35-1841edefa459 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.450498615Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.450732428Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.450787439Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/4859e6da-cd85-4148-ad35-1841edefa459","method":"DELETE","req_id":"9e135c56-1ac5-415f-8c6f-274eed8c7fa3","remote_addr":"[fd00:1122:3344:101::a]:63168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":19489,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-4859e6da-cd85-4148-ad35-1841edefa459 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.452646509Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.454123132Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/4859e6da-cd85-4148-ad35-1841edefa459","method":"GET","req_id":"783ab0fa-e2e1-4f5c-b59d-92b98b8a22ff","remote_addr":"[fd00:1122:3344:101::a]:63168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.458015825Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_06c5d615-3101-4280-b654-568002756968/crucible/regions/4859e6da-cd85-4148-ad35-1841edefa459\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.467329234Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"region":"4859e6da-cd85-4148-ad35-1841edefa459","component":"worker"}
{"msg":"region 4859e6da-cd85-4148-ad35-1841edefa459 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.514017069Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-25T04:56:11.553638171Z","hostname":"oxz_crucible_d22809a8-5081-49ac-9d75-9080ffa4899b","pid":22779,"uri":"/crucible/0/regions/4859e6da-cd85-4148-ad35-1841edefa459","method":"GET","req_id":"36e12e63-cd40-47bf-ad1a-202762884ef5","remote_addr":"[fd00:1122:3344:101::a]:63168","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":64,"response_code":"200"}