{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.218601318Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54538"} {"msg":"region caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.218688979Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.219609274Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions","method":"POST","req_id":"3903aefc-07f2-4d6b-8a29-c64a8c63a70d","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":942,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.219621704Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.230426124Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.230456195Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.281885392Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"region":"caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb\" \"--data\" \"/data/regions/caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.281939713Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"region":"caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.353172939Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"region":"caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb","component":"worker"} {"msg":"region caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.353256201Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.353519635Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.372567602Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.372680744Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.372781726Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.372789506Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.372843977Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.372920648Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.37303991Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.373122861Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.373171972Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.373224283Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.385847064Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.38625552Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.388419146Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.395126628Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.398905341Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/caadc3dc-c3bf-4fc9-be5f-74439dfb7ddb","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.399170446Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.399244437Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.399288837Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.399331138Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.402498311Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.406887184Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:40.444069443Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions","method":"POST","req_id":"dd415282-6324-4f92-8851-d8812cc1235d","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":65,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.366415628Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions/22857358-599b-4805-a7cd-5ffd15fc8459","method":"GET","req_id":"8816329d-23dc-4a21-b156-a97207c740ab","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":4959,"response_code":"200"} {"msg":"path is \"/data/regions/22857358-599b-4805-a7cd-5ffd15fc8459\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.369375787Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"dataset is oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.378390067Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"snapshot is oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/22857358-599b-4805-a7cd-5ffd15fc8459@3b4fc648-830d-4e3d-8847-e70e395f0e3d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.38578845Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/22857358-599b-4805-a7cd-5ffd15fc8459\", \"3b4fc648-830d-4e3d-8847-e70e395f0e3d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.385812841Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.396780334Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions/22857358-599b-4805-a7cd-5ffd15fc8459/snapshots/3b4fc648-830d-4e3d-8847-e70e395f0e3d","method":"GET","req_id":"fd991710-8799-4753-9861-f8a50f907e80","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":27407,"response_code":"200"} {"msg":"path is \"/data/regions/22857358-599b-4805-a7cd-5ffd15fc8459\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.402417218Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"dataset is oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.409927493Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"snapshot is oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/22857358-599b-4805-a7cd-5ffd15fc8459@3b4fc648-830d-4e3d-8847-e70e395f0e3d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.418830441Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/22857358-599b-4805-a7cd-5ffd15fc8459\", \"3b4fc648-830d-4e3d-8847-e70e395f0e3d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.418850921Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 22857358-599b-4805-a7cd-5ffd15fc8459-3b4fc648-830d-4e3d-8847-e70e395f0e3d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.428537623Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"applying SMF actions for region 22857358-599b-4805-a7cd-5ffd15fc8459 running snapshot 3b4fc648-830d-4e3d-8847-e70e395f0e3d (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.428728686Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.428750066Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions/22857358-599b-4805-a7cd-5ffd15fc8459/snapshots/3b4fc648-830d-4e3d-8847-e70e395f0e3d/run","method":"POST","req_id":"835ac632-fa55-4856-b53d-2a28e77aefec","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":26365,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.429306855Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.429392807Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.429472488Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.429483038Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.429519799Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.42956543Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.429940276Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.430001957Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.430049138Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.430094299Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.43379607Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-22857358-599b-4805-a7cd-5ffd15fc8459-3b4fc648-830d-4e3d-8847-e70e395f0e3d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.435250474Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-22857358-599b-4805-a7cd-5ffd15fc8459-3b4fc648-830d-4e3d-8847-e70e395f0e3d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.442611917Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-22857358-599b-4805-a7cd-5ffd15fc8459-3b4fc648-830d-4e3d-8847-e70e395f0e3d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.451231751Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/22857358-599b-4805-a7cd-5ffd15fc8459/.zfs/snapshot/3b4fc648-830d-4e3d-8847-e70e395f0e3d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.451480875Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.451549516Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.451588187Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.451630807Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.451692499Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.455460101Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.459984317Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"region 22857358-599b-4805-a7cd-5ffd15fc8459 running snapshot 3b4fc648-830d-4e3d-8847-e70e395f0e3d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.459995097Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"path is \"/data/regions/22857358-599b-4805-a7cd-5ffd15fc8459\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.560255407Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"dataset is oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.569198276Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"snapshot is oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/22857358-599b-4805-a7cd-5ffd15fc8459@3b4fc648-830d-4e3d-8847-e70e395f0e3d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.577199789Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/22857358-599b-4805-a7cd-5ffd15fc8459\", \"3b4fc648-830d-4e3d-8847-e70e395f0e3d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.57722758Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:41.58507895Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions/22857358-599b-4805-a7cd-5ffd15fc8459/snapshots/3b4fc648-830d-4e3d-8847-e70e395f0e3d/run","method":"POST","req_id":"0b703e77-a1ad-4532-9383-f9d1d3ef4eaf","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":24874,"response_code":"200"} {"msg":"region 48845bad-fba5-419c-b732-f7e97df8b466 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.011189373Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.011370225Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions","method":"POST","req_id":"f65412be-72be-4baa-b759-f3ec3f4a2fa7","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":249,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.011390666Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/48845bad-fba5-419c-b732-f7e97df8b466","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.021742518Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/48845bad-fba5-419c-b732-f7e97df8b466","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.021776279Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"48845bad-fba5-419c-b732-f7e97df8b466\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/48845bad-fba5-419c-b732-f7e97df8b466\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.091959728Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"region":"48845bad-fba5-419c-b732-f7e97df8b466","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"48845bad-fba5-419c-b732-f7e97df8b466\" \"--data\" \"/data/regions/48845bad-fba5-419c-b732-f7e97df8b466\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.092017439Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"region":"48845bad-fba5-419c-b732-f7e97df8b466","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.161189301Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"region":"48845bad-fba5-419c-b732-f7e97df8b466","component":"worker"} {"msg":"region 48845bad-fba5-419c-b732-f7e97df8b466 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.161236742Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.161329424Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.161929294Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.162054366Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.162155757Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.162168308Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.162209408Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.162270109Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.162636636Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.162702906Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.162750917Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.162812048Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.167158661Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-48845bad-fba5-419c-b732-f7e97df8b466","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.167620699Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-48845bad-fba5-419c-b732-f7e97df8b466","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.170893173Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.177803918Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-48845bad-fba5-419c-b732-f7e97df8b466","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.180208288Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/48845bad-fba5-419c-b732-f7e97df8b466","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.180524663Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.180562834Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.180610715Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.180654316Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.184317466Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.195597454Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:54:45.28655029Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions","method":"POST","req_id":"72286116-472a-48d1-bcf0-d000661ffa27","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.901198501Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions/48845bad-fba5-419c-b732-f7e97df8b466","method":"GET","req_id":"edd62cfe-8d3f-42ca-ae22-0a77c238d94d","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":69,"response_code":"200"} {"msg":"path is \"/data/regions/48845bad-fba5-419c-b732-f7e97df8b466\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.905162477Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"dataset is oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/48845bad-fba5-419c-b732-f7e97df8b466","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.916974504Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"region 48845bad-fba5-419c-b732-f7e97df8b466 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.926404981Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.926626815Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-48845bad-fba5-419c-b732-f7e97df8b466 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.927838695Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.929144937Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions/48845bad-fba5-419c-b732-f7e97df8b466","method":"DELETE","req_id":"fc13b583-dc8f-4e9e-b8ef-890d642fc4f0","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":24005,"response_code":"204"} {"msg":"existing directory value /dev/null does not match /data/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.932694706Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.932758087Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions/48845bad-fba5-419c-b732-f7e97df8b466","method":"GET","req_id":"7ddd57ef-e4dd-467b-91ef-4299a06da473","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":33,"response_code":"200"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.932835948Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.932976211Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.932987151Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.933048932Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.933115773Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/22857358-599b-4805-a7cd-5ffd15fc8459","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.933579191Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.933656072Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.933724163Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.933804004Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.938216488Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.940522826Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_9ad521f2-498d-459f-9590-bf571f0f7fe6/crucible/regions/48845bad-fba5-419c-b732-f7e97df8b466\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:09.952696889Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"region":"48845bad-fba5-419c-b732-f7e97df8b466","component":"worker"} {"msg":"region 48845bad-fba5-419c-b732-f7e97df8b466 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:10.009688559Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:55:10.029987587Z","hostname":"oxz_crucible_8d44a8ba-e3ce-4ce4-bf93-31cc8e795fed","pid":23309,"uri":"/crucible/0/regions/48845bad-fba5-419c-b732-f7e97df8b466","method":"GET","req_id":"ba19ec4f-0fbf-4457-a868-bc876909cb9f","remote_addr":"[fd00:1122:3344:101::a]:54538","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":83,"response_code":"200"}