[ Jul  9 03:56:27 Enabled. ]
[ Jul  9 03:56:27 Rereading configuration. ]
[ Jul  9 03:56:28 Rereading configuration. ]
[ Jul  9 03:56:31 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6465aeea-d024-4ee4-a24b-ff08e90708a1/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_6465aeea-d024-4ee4-a24b-ff08e90708a1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:56:31.188099437Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:56:31.198321569Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:56:31.198338717Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:56:31.282373704Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:56:31.468388011Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.683178294Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46140"}
{"msg":"region 6c174b78-12e7-4366-8d6e-be7cd0a97601 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.684540877Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.686872037Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"uri":"/crucible/0/regions","method":"POST","req_id":"aebd92c3-2ef1-493a-b1fb-03c1c7e6619c","remote_addr":"[fd00:1122:3344:101::a]:46140","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":2641,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.687574362Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_6465aeea-d024-4ee4-a24b-ff08e90708a1/crucible/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.697110178Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_6465aeea-d024-4ee4-a24b-ff08e90708a1/crucible/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.69714016Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6c174b78-12e7-4366-8d6e-be7cd0a97601\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.756333268Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"region":"6c174b78-12e7-4366-8d6e-be7cd0a97601","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6c174b78-12e7-4366-8d6e-be7cd0a97601\" \"--data\" \"/data/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.756386631Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"region":"6c174b78-12e7-4366-8d6e-be7cd0a97601","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.86459406Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"region":"6c174b78-12e7-4366-8d6e-be7cd0a97601","component":"worker"}
{"msg":"region 6c174b78-12e7-4366-8d6e-be7cd0a97601 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.864638939Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.864722575Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6c174b78-12e7-4366-8d6e-be7cd0a97601","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.8652659Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6c174b78-12e7-4366-8d6e-be7cd0a97601","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.868848401Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.876571418Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.876659376Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"uri":"/crucible/0/regions","method":"POST","req_id":"32625cd4-0bae-4091-a1e7-dd7fa15e26f5","remote_addr":"[fd00:1122:3344:101::a]:46140","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":652,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c174b78-12e7-4366-8d6e-be7cd0a97601","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.879295874Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.880080147Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.882188825Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.882242348Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.882280833Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.891009907Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:10.89764108Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.576317167Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"uri":"/crucible/0/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601","method":"GET","req_id":"cd8d4db6-dd02-442c-b3ef-a8dfc75775be","remote_addr":"[fd00:1122:3344:101::a]:46140","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"path is \"/data/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.57960164Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"datafile"}
{"msg":"dataset is oxp_6465aeea-d024-4ee4-a24b-ff08e90708a1/crucible/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.588605111Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"datafile"}
{"msg":"region 6c174b78-12e7-4366-8d6e-be7cd0a97601 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.59674978Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.596979797Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.597027151Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"uri":"/crucible/0/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601","method":"DELETE","req_id":"a9ee601c-b8df-4de1-8f4b-132f7f1a126f","remote_addr":"[fd00:1122:3344:101::a]:46140","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18048,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-6c174b78-12e7-4366-8d6e-be7cd0a97601 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.598289031Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.60061134Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"uri":"/crucible/0/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601","method":"GET","req_id":"ff7e045b-4823-4dad-aaee-b18e4612a28d","remote_addr":"[fd00:1122:3344:101::a]:46140","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.602483313Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_6465aeea-d024-4ee4-a24b-ff08e90708a1/crucible/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.611375913Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"region":"6c174b78-12e7-4366-8d6e-be7cd0a97601","component":"worker"}
{"msg":"region 6c174b78-12e7-4366-8d6e-be7cd0a97601 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.661748563Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:11.72816132Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"uri":"/crucible/0/regions/6c174b78-12e7-4366-8d6e-be7cd0a97601","method":"GET","req_id":"0c4ee0a3-cc6d-4ccf-8591-1613c7498ba0","remote_addr":"[fd00:1122:3344:101::a]:46140","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"region 1701aba8-1c09-4b05-8dda-08579865b105 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.490097406Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.492160718Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.492177004Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"uri":"/crucible/0/regions","method":"POST","req_id":"56971a1d-e86e-475e-9f63-6df883c2e024","remote_addr":"[fd00:1122:3344:101::a]:46140","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":2936,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_6465aeea-d024-4ee4-a24b-ff08e90708a1/crucible/regions/1701aba8-1c09-4b05-8dda-08579865b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.501421488Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_6465aeea-d024-4ee4-a24b-ff08e90708a1/crucible/regions/1701aba8-1c09-4b05-8dda-08579865b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.501455725Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1701aba8-1c09-4b05-8dda-08579865b105\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1701aba8-1c09-4b05-8dda-08579865b105\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.572864402Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"region":"1701aba8-1c09-4b05-8dda-08579865b105","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1701aba8-1c09-4b05-8dda-08579865b105\" \"--data\" \"/data/regions/1701aba8-1c09-4b05-8dda-08579865b105\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.575396723Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"region":"1701aba8-1c09-4b05-8dda-08579865b105","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.64318383Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"region":"1701aba8-1c09-4b05-8dda-08579865b105","component":"worker"}
{"msg":"region 1701aba8-1c09-4b05-8dda-08579865b105 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.643268442Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.64349991Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1701aba8-1c09-4b05-8dda-08579865b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.644842718Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1701aba8-1c09-4b05-8dda-08579865b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.647916023Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.656159433Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1701aba8-1c09-4b05-8dda-08579865b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.659031426Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1701aba8-1c09-4b05-8dda-08579865b105","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.659318493Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.659374321Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.659493161Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.659533466Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.663609064Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.669058218Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-09T03:57:13.749248626Z","hostname":"oxz_crucible_39cee393-d680-4c39-887a-475060dc671f","pid":22853,"uri":"/crucible/0/regions","method":"POST","req_id":"8e37eb56-7ebe-4d28-8545-9f4cd3937ecd","remote_addr":"[fd00:1122:3344:101::a]:46140","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"}