[ Jun 17 18:06:39 Enabled. ]
[ Jun 17 18:06:39 Rereading configuration. ]
[ Jun 17 18:06:41 Rereading configuration. ]
[ Jun 17 18:06:42 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_df85cc24-f970-4f8d-b0e2-4f3c27577b17/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_df85cc24-f970-4f8d-b0e2-4f3c27577b17/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:06:42.765759021Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:06:42.766051358Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:06:42.766061028Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:06:42.782053758Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:06:42.848584047Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.63403031Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:50879"}
{"msg":"region 57db79ac-91b4-4800-8bf8-3dea5c3301da state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.649885826Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.650983653Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.654908349Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"uri":"/crucible/0/regions","method":"POST","req_id":"6ef805e3-8c42-48e3-9765-44c82df9881c","remote_addr":"[fd00:1122:3344:101::a]:50879","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":11794,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_df85cc24-f970-4f8d-b0e2-4f3c27577b17/crucible/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.66069569Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_df85cc24-f970-4f8d-b0e2-4f3c27577b17/crucible/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.660738511Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"57db79ac-91b4-4800-8bf8-3dea5c3301da\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.72070518Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"region":"57db79ac-91b4-4800-8bf8-3dea5c3301da","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"57db79ac-91b4-4800-8bf8-3dea5c3301da\" \"--data\" \"/data/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.722626067Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"region":"57db79ac-91b4-4800-8bf8-3dea5c3301da","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.865038293Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"region":"57db79ac-91b4-4800-8bf8-3dea5c3301da","component":"worker"}
{"msg":"region 57db79ac-91b4-4800-8bf8-3dea5c3301da state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.865113385Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.86530127Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.8657116Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.870862405Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.885394899Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.888890683Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.889478977Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.889963729Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"uri":"/crucible/0/regions","method":"POST","req_id":"7920bbd1-f260-4e45-b180-e2c753245b0c","remote_addr":"[fd00:1122:3344:101::a]:50879","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":672,"response_code":"200"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.891601099Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.89164672Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.891688531Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.900082305Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:07:07.90481046Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.162126504Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61232"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.166752557Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"uri":"/crucible/0/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da","method":"GET","req_id":"9d810283-ead4-4ea8-b649-f5767523efeb","remote_addr":"[fd00:1122:3344:101::a]:61232","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":4581,"response_code":"200"}
{"msg":"path is \"/data/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.167085945Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"datafile"}
{"msg":"dataset is oxp_df85cc24-f970-4f8d-b0e2-4f3c27577b17/crucible/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.178137844Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"datafile"}
{"msg":"snapshot is oxp_df85cc24-f970-4f8d-b0e2-4f3c27577b17/crucible/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da@33f80c78-d70c-4161-b1d0-689787d4000d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.188712531Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_df85cc24-f970-4f8d-b0e2-4f3c27577b17/crucible/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da\", \"33f80c78-d70c-4161-b1d0-689787d4000d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.188751432Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.206076614Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"uri":"/crucible/0/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da/snapshots/33f80c78-d70c-4161-b1d0-689787d4000d","method":"GET","req_id":"445c5fda-319a-434e-ad7c-83f51ffbb3a1","remote_addr":"[fd00:1122:3344:101::a]:61232","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":39043,"response_code":"200"}
{"msg":"path is \"/data/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da\"","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.20631254Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"datafile"}
{"msg":"dataset is oxp_df85cc24-f970-4f8d-b0e2-4f3c27577b17/crucible/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.215856562Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"datafile"}
{"msg":"snapshot is oxp_df85cc24-f970-4f8d-b0e2-4f3c27577b17/crucible/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da@33f80c78-d70c-4161-b1d0-689787d4000d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.223817455Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_df85cc24-f970-4f8d-b0e2-4f3c27577b17/crucible/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da\", \"33f80c78-d70c-4161-b1d0-689787d4000d\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.223873727Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 57db79ac-91b4-4800-8bf8-3dea5c3301da-33f80c78-d70c-4161-b1d0-689787d4000d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.233531782Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"datafile"}
{"msg":"applying SMF actions for region 57db79ac-91b4-4800-8bf8-3dea5c3301da running snapshot 33f80c78-d70c-4161-b1d0-689787d4000d (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.234084685Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.234135287Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"uri":"/crucible/0/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da/snapshots/33f80c78-d70c-4161-b1d0-689787d4000d/run","method":"POST","req_id":"3feca08e-3470-4787-a3a9-268e045e24be","remote_addr":"[fd00:1122:3344:101::a]:61232","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":27856,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.24043239Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.240513582Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.240595684Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.240602724Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.240644515Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.240691696Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.240781819Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.24084217Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.240890051Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.240950543Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.246415876Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-57db79ac-91b4-4800-8bf8-3dea5c3301da-33f80c78-d70c-4161-b1d0-689787d4000d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.246880317Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-57db79ac-91b4-4800-8bf8-3dea5c3301da-33f80c78-d70c-4161-b1d0-689787d4000d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.25029141Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-57db79ac-91b4-4800-8bf8-3dea5c3301da-33f80c78-d70c-4161-b1d0-689787d4000d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.262521037Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/57db79ac-91b4-4800-8bf8-3dea5c3301da/.zfs/snapshot/33f80c78-d70c-4161-b1d0-689787d4000d","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.262814355Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.262870186Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.262914077Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.262954348Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.263001849Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.266251528Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.270620285Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"worker"}
{"msg":"region 57db79ac-91b4-4800-8bf8-3dea5c3301da running snapshot 33f80c78-d70c-4161-b1d0-689787d4000d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-06-17T18:12:25.270651306Z","hostname":"oxz_crucible_c3407aab-c332-4f05-9812-dd83f50d6013","pid":23244,"component":"datafile"}