{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.287308988Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33091"} {"msg":"region bc06ce06-8021-4bc7-b0ba-7303fac193fb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.28740055Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.288285694Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.288295825Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions","method":"POST","req_id":"026eebe8-089d-4502-8f6a-e9d33d35f9d9","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":936,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/bc06ce06-8021-4bc7-b0ba-7303fac193fb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.300225308Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/bc06ce06-8021-4bc7-b0ba-7303fac193fb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.300248548Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"bc06ce06-8021-4bc7-b0ba-7303fac193fb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bc06ce06-8021-4bc7-b0ba-7303fac193fb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.356184253Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"region":"bc06ce06-8021-4bc7-b0ba-7303fac193fb","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bc06ce06-8021-4bc7-b0ba-7303fac193fb\" \"--data\" \"/data/regions/bc06ce06-8021-4bc7-b0ba-7303fac193fb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.356236754Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"region":"bc06ce06-8021-4bc7-b0ba-7303fac193fb","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.436243221Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"region":"bc06ce06-8021-4bc7-b0ba-7303fac193fb","component":"worker"} {"msg":"region bc06ce06-8021-4bc7-b0ba-7303fac193fb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.436279082Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.436387995Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.447493626Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.447601079Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.447700022Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.447709172Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.447758013Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.447828935Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.447938668Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.44801233Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.448068872Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.448126083Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.455894553Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-bc06ce06-8021-4bc7-b0ba-7303fac193fb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.456293494Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bc06ce06-8021-4bc7-b0ba-7303fac193fb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.458878484Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.468063023Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bc06ce06-8021-4bc7-b0ba-7303fac193fb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.470737975Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bc06ce06-8021-4bc7-b0ba-7303fac193fb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.470937161Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.470987112Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.471024983Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.471061874Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.474047475Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.47791266Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:37.594394435Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions","method":"POST","req_id":"6c22915a-12ed-473e-adea-1c05ab8d5765","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":67,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.823174643Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","method":"GET","req_id":"bcf6d2f5-e88c-4487-b060-bd00331c7b19","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":4596,"response_code":"200"} {"msg":"path is \"/data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.826779071Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"dataset is oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.835144118Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"snapshot is oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b@a1495436-e73c-4083-a306-302f164a9736","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.842589699Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b\", \"a1495436-e73c-4083-a306-302f164a9736\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.84262109Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.851797238Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b/snapshots/a1495436-e73c-4083-a306-302f164a9736","method":"GET","req_id":"100cc27c-14ca-4422-9738-8f9b41cbbca0","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25034,"response_code":"200"} {"msg":"path is \"/data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.855273033Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"dataset is oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.862122558Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"snapshot is oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b@a1495436-e73c-4083-a306-302f164a9736","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.868921032Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b\", \"a1495436-e73c-4083-a306-302f164a9736\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.868948593Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b-a1495436-e73c-4083-a306-302f164a9736 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.876392475Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"applying SMF actions for region a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b running snapshot a1495436-e73c-4083-a306-302f164a9736 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.87657705Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.876625881Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b/snapshots/a1495436-e73c-4083-a306-302f164a9736/run","method":"POST","req_id":"a8093e07-208f-4a91-b90a-740a0e83118e","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21387,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.877255868Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.877353981Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.877450973Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.877464124Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.877499695Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.877544266Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.877907656Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.877966737Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.878012259Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.87805512Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.881220435Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b-a1495436-e73c-4083-a306-302f164a9736","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.882777198Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b-a1495436-e73c-4083-a306-302f164a9736","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.89025804Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b-a1495436-e73c-4083-a306-302f164a9736","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.898831032Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b/.zfs/snapshot/a1495436-e73c-4083-a306-302f164a9736","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.899021488Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.899069509Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.89910731Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.899142481Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.899184852Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.90280713Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.907057605Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"region a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b running snapshot a1495436-e73c-4083-a306-302f164a9736 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:38.907067706Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"path is \"/data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:39.054706204Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"dataset is oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:39.06415188Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"snapshot is oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b@a1495436-e73c-4083-a306-302f164a9736","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:39.0722752Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b\", \"a1495436-e73c-4083-a306-302f164a9736\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:39.072308411Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:39.081103359Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b/snapshots/a1495436-e73c-4083-a306-302f164a9736/run","method":"POST","req_id":"970574f8-2304-4a4e-92ba-5c5ff191e12b","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":27048,"response_code":"200"} {"msg":"region ca2a95ea-962f-416d-98f8-5491942c19d7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.431628041Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.431802656Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions","method":"POST","req_id":"770e5930-5c28-4457-bc7a-16ec1774471f","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":235,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.431843587Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/ca2a95ea-962f-416d-98f8-5491942c19d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.442941038Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/ca2a95ea-962f-416d-98f8-5491942c19d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.442979839Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"ca2a95ea-962f-416d-98f8-5491942c19d7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ca2a95ea-962f-416d-98f8-5491942c19d7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.514367082Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"region":"ca2a95ea-962f-416d-98f8-5491942c19d7","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ca2a95ea-962f-416d-98f8-5491942c19d7\" \"--data\" \"/data/regions/ca2a95ea-962f-416d-98f8-5491942c19d7\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.514421994Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"region":"ca2a95ea-962f-416d-98f8-5491942c19d7","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.597903415Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"region":"ca2a95ea-962f-416d-98f8-5491942c19d7","component":"worker"} {"msg":"region ca2a95ea-962f-416d-98f8-5491942c19d7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.597937926Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.598022058Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.598640655Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.598720717Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.598798869Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.598809519Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.59884478Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.598901712Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.599337904Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.599420776Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.599472787Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.599523998Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.605095329Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-ca2a95ea-962f-416d-98f8-5491942c19d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.606262921Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.608615645Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions","method":"POST","req_id":"82f2bcab-a66c-4528-bfe0-4834f9c9b5a3","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":112,"response_code":"200"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ca2a95ea-962f-416d-98f8-5491942c19d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.609416926Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.617617879Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ca2a95ea-962f-416d-98f8-5491942c19d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.620425415Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ca2a95ea-962f-416d-98f8-5491942c19d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.620704712Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.620770934Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.620810305Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.620850726Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.624386422Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:52:42.630312322Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.063797003Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions/ca2a95ea-962f-416d-98f8-5491942c19d7","method":"GET","req_id":"74bf872d-33ff-4b6c-a837-3123bd847470","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":62,"response_code":"200"} {"msg":"path is \"/data/regions/ca2a95ea-962f-416d-98f8-5491942c19d7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.067912285Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"dataset is oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/ca2a95ea-962f-416d-98f8-5491942c19d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.080407533Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"region ca2a95ea-962f-416d-98f8-5491942c19d7 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.090355793Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.090570429Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-ca2a95ea-962f-416d-98f8-5491942c19d7 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.091908845Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.092985014Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions/ca2a95ea-962f-416d-98f8-5491942c19d7","method":"DELETE","req_id":"06ae00ea-d1b9-4bbf-abd6-cf4c56117bb9","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25100,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.096393536Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions/ca2a95ea-962f-416d-98f8-5491942c19d7","method":"GET","req_id":"843ddd78-10e4-4591-b727-ee94e37de119","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":31,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.097534497Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.09762517Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.097714022Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.097725893Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.097774014Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.097870886Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a9483005-afd7-4c3d-9ab7-4bf3a1b47e5b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.098168814Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.098237218Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.098296086Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.098349178Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.102262045Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.104560627Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_f1d7fa4a-6150-4e79-a410-b71fd3b63c5c/crucible/regions/ca2a95ea-962f-416d-98f8-5491942c19d7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.115253757Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"region":"ca2a95ea-962f-416d-98f8-5491942c19d7","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.18402782Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions/ca2a95ea-962f-416d-98f8-5491942c19d7","method":"GET","req_id":"a8a5977b-e07f-403d-aa02-93e469893c5a","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":1233,"response_code":"200"} {"msg":"region ca2a95ea-962f-416d-98f8-5491942c19d7 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.196389474Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T19:53:07.34464889Z","hostname":"oxz_crucible_d2869dcb-02ac-4b14-a79c-5e73e9f82137","pid":22956,"uri":"/crucible/0/regions/ca2a95ea-962f-416d-98f8-5491942c19d7","method":"GET","req_id":"3276d6a6-3cd2-4104-baf5-136482ca331d","remote_addr":"[fd00:1122:3344:101::a]:33091","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":34,"response_code":"200"}