{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.033623417Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:37279"} {"msg":"region b23751d0-b7f1-4193-9d0a-5f0d4c8737aa state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.033702329Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.037221303Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.037238643Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions","method":"POST","req_id":"3a350af8-cc3b-4766-ab2e-1212044200a9","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":3545,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/b23751d0-b7f1-4193-9d0a-5f0d4c8737aa","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.04909472Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/b23751d0-b7f1-4193-9d0a-5f0d4c8737aa","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.049125361Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"b23751d0-b7f1-4193-9d0a-5f0d4c8737aa\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b23751d0-b7f1-4193-9d0a-5f0d4c8737aa\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.110329412Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"region":"b23751d0-b7f1-4193-9d0a-5f0d4c8737aa","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b23751d0-b7f1-4193-9d0a-5f0d4c8737aa\" \"--data\" \"/data/regions/b23751d0-b7f1-4193-9d0a-5f0d4c8737aa\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.110381043Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"region":"b23751d0-b7f1-4193-9d0a-5f0d4c8737aa","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.194586116Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"region":"b23751d0-b7f1-4193-9d0a-5f0d4c8737aa","component":"worker"} {"msg":"region b23751d0-b7f1-4193-9d0a-5f0d4c8737aa state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.194638946Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.194737228Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-b23751d0-b7f1-4193-9d0a-5f0d4c8737aa","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.195199267Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b23751d0-b7f1-4193-9d0a-5f0d4c8737aa","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.200970342Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.214910218Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b23751d0-b7f1-4193-9d0a-5f0d4c8737aa","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.218751628Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b23751d0-b7f1-4193-9d0a-5f0d4c8737aa","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.219050784Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.219105895Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.219148335Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.219189866Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.222978346Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.22977915Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.229865752Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.229949783Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.229957684Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.229999404Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.230045655Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.230122547Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.230178657Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.230227469Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.230273889Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.243043883Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.243511302Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:47.283075757Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions","method":"POST","req_id":"5283b91e-c1cd-4c85-9ffb-830cd2bc879e","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":62,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.00461514Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","method":"GET","req_id":"7cd0f303-1ae3-47a9-8430-9b7a3520ca53","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":585,"response_code":"200"} {"msg":"path is \"/data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.008726095Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"dataset is oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.018864921Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"snapshot is oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67@543dd394-4a17-4bc3-a423-ca149df40408","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.027889286Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67\", \"543dd394-4a17-4bc3-a423-ca149df40408\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.027926747Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.038870737Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67/snapshots/543dd394-4a17-4bc3-a423-ca149df40408","method":"GET","req_id":"cb3f57e0-a002-40f1-89ed-d0b020371d79","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":30148,"response_code":"200"} {"msg":"path is \"/data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.042407022Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"dataset is oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.051492408Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"snapshot is oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67@543dd394-4a17-4bc3-a423-ca149df40408","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.060005844Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67\", \"543dd394-4a17-4bc3-a423-ca149df40408\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.060035595Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot d94f46df-aaac-4329-b2ca-8c7884c61d67-543dd394-4a17-4bc3-a423-ca149df40408 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.069110851Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"applying SMF actions for region d94f46df-aaac-4329-b2ca-8c7884c61d67 running snapshot 543dd394-4a17-4bc3-a423-ca149df40408 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.069357425Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.069370306Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67/snapshots/543dd394-4a17-4bc3-a423-ca149df40408/run","method":"POST","req_id":"f7d57c2f-0957-4e11-b63a-1606433b89a3","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":26978,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.071039336Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.071122538Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.071202149Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.071212219Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.07125084Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.071295451Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.071607357Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.071666768Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.071714939Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.071760729Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.075795263Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-d94f46df-aaac-4329-b2ca-8c7884c61d67-543dd394-4a17-4bc3-a423-ca149df40408","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.076289803Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-d94f46df-aaac-4329-b2ca-8c7884c61d67-543dd394-4a17-4bc3-a423-ca149df40408","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.084982402Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-d94f46df-aaac-4329-b2ca-8c7884c61d67-543dd394-4a17-4bc3-a423-ca149df40408","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.096548954Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67/.zfs/snapshot/543dd394-4a17-4bc3-a423-ca149df40408","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.096751207Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.096811208Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.096852369Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.09689314Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.096942681Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.10074926Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.105918715Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"region d94f46df-aaac-4329-b2ca-8c7884c61d67 running snapshot 543dd394-4a17-4bc3-a423-ca149df40408 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.105928855Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"path is \"/data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.382789557Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"dataset is oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.391112389Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"snapshot is oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67@543dd394-4a17-4bc3-a423-ca149df40408","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.399385341Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67\", \"543dd394-4a17-4bc3-a423-ca149df40408\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.399431622Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:49.408045069Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67/snapshots/543dd394-4a17-4bc3-a423-ca149df40408/run","method":"POST","req_id":"9bb49397-1f5a-41c4-ad3e-3c0933facf63","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":25256,"response_code":"200"} {"msg":"region 4e900422-c320-4ad9-8b8b-1434ec3cdeb4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.32955125Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.329701662Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions","method":"POST","req_id":"a7e923ee-bcf2-4acb-bd56-92002d9e4741","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":750,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.329725623Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.339457671Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.339500382Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"4e900422-c320-4ad9-8b8b-1434ec3cdeb4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.408004027Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"region":"4e900422-c320-4ad9-8b8b-1434ec3cdeb4","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4e900422-c320-4ad9-8b8b-1434ec3cdeb4\" \"--data\" \"/data/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.412068801Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"region":"4e900422-c320-4ad9-8b8b-1434ec3cdeb4","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.500809476Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"region":"4e900422-c320-4ad9-8b8b-1434ec3cdeb4","component":"worker"} {"msg":"region 4e900422-c320-4ad9-8b8b-1434ec3cdeb4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.500854487Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.50097553Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-4e900422-c320-4ad9-8b8b-1434ec3cdeb4","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.5015682Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4e900422-c320-4ad9-8b8b-1434ec3cdeb4","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.505089465Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.512803406Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4e900422-c320-4ad9-8b8b-1434ec3cdeb4","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.51572926Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.515942064Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.515998505Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.516041135Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.516080446Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.5195466Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.525292235Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.525408607Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.525518379Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.525533169Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.52558488Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.525643971Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.529676445Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.529733816Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.529782737Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.529829408Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.533819301Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.535750456Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:22:53.639901904Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions","method":"POST","req_id":"79335148-f894-40e0-83a0-8fe536e1ae27","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":77,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.463446007Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4","method":"GET","req_id":"bed4d0f8-b1f0-40e6-816b-1899cd516acb","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":44,"response_code":"200"} {"msg":"path is \"/data/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.46692787Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"dataset is oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.480151002Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"region 4e900422-c320-4ad9-8b8b-1434ec3cdeb4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.490054284Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.490230857Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-4e900422-c320-4ad9-8b8b-1434ec3cdeb4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.491356878Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.492504819Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4","method":"DELETE","req_id":"aac7d639-dcd0-414f-93be-fb066e89311c","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":25601,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.495576945Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4","method":"GET","req_id":"f6d3e304-4c14-4ec6-9bc7-5783287252d4","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.498140332Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.498239534Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.498323646Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.498331735Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.498377367Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.498422787Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d94f46df-aaac-4329-b2ca-8c7884c61d67","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.498886736Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.498945967Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.498993618Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.499039568Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.503314027Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.504881046Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_5286a236-bcd6-4953-a6e4-4910373227fe/crucible/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.51605078Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"region":"4e900422-c320-4ad9-8b8b-1434ec3cdeb4","component":"worker"} {"msg":"region 4e900422-c320-4ad9-8b8b-1434ec3cdeb4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.562549392Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-13T06:23:18.6022842Z","hostname":"oxz_crucible_26ed89ba-2f36-41d8-ba28-a0e7d0715eb0","pid":22192,"uri":"/crucible/0/regions/4e900422-c320-4ad9-8b8b-1434ec3cdeb4","method":"GET","req_id":"20ca67dd-9bd2-42cc-b92c-663f187fa9e5","remote_addr":"[fd00:1122:3344:101::b]:37279","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":67,"response_code":"200"}