[ Dec 21 00:47:34 Enabled. ]
[ Dec 21 00:47:34 Rereading configuration. ]
[ Dec 21 00:47:36 Rereading configuration. ]
[ Dec 21 00:47:40 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:47:40.622365177Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:47:40.625895092Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:47:40.625904972Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:47:40.645910684Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:47:40.750529395Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.689982309Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48264"}
{"msg":"region 1d39c011-f797-499f-8954-1f1d41751e5f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.723351888Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.727544456Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.732140161Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"uri":"/crucible/0/regions","method":"POST","req_id":"38afb235-a77d-4951-9305-1a850d321cbc","remote_addr":"[fd00:1122:3344:101::a]:48264","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":14645,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.7391105Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.739131761Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1d39c011-f797-499f-8954-1f1d41751e5f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1d39c011-f797-499f-8954-1f1d41751e5f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.802526447Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"region":"1d39c011-f797-499f-8954-1f1d41751e5f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1d39c011-f797-499f-8954-1f1d41751e5f\" \"--data\" \"/data/regions/1d39c011-f797-499f-8954-1f1d41751e5f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.804459843Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"region":"1d39c011-f797-499f-8954-1f1d41751e5f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.942940993Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"region":"1d39c011-f797-499f-8954-1f1d41751e5f","component":"worker"}
{"msg":"region 1d39c011-f797-499f-8954-1f1d41751e5f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.943004004Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.943073985Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1d39c011-f797-499f-8954-1f1d41751e5f","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.943474573Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1d39c011-f797-499f-8954-1f1d41751e5f","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.951333669Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.961016608Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"uri":"/crucible/0/regions","method":"POST","req_id":"85734f37-de07-4734-9e42-a3fed9486c21","remote_addr":"[fd00:1122:3344:101::a]:48264","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.96543411Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1d39c011-f797-499f-8954-1f1d41751e5f","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.970461144Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1d39c011-f797-499f-8954-1f1d41751e5f","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.971170297Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.971241878Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.971290179Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.97133808Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.976444315Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:48:08.981846965Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.026691491Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44881"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.033027819Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"uri":"/crucible/0/regions/1d39c011-f797-499f-8954-1f1d41751e5f","method":"GET","req_id":"6a16b2a9-81cd-4f4d-b974-e58b3eed1809","remote_addr":"[fd00:1122:3344:101::a]:44881","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":6289,"response_code":"200"}
{"msg":"path is \"/data/regions/1d39c011-f797-499f-8954-1f1d41751e5f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.035987383Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"dataset is oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.045707614Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"snapshot is oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f@99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.054820373Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f\", \"99d9a509-237d-43cd-9c13-86aab0a26b82\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.054850353Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.071034824Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"uri":"/crucible/0/regions/1d39c011-f797-499f-8954-1f1d41751e5f/snapshots/99d9a509-237d-43cd-9c13-86aab0a26b82","method":"GET","req_id":"960d6663-9d6d-46d5-bc4b-3aa9a69f609c","remote_addr":"[fd00:1122:3344:101::a]:44881","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":35068,"response_code":"200"}
{"msg":"path is \"/data/regions/1d39c011-f797-499f-8954-1f1d41751e5f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.075431395Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"dataset is oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.083633538Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"snapshot is oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f@99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.092266258Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f\", \"99d9a509-237d-43cd-9c13-86aab0a26b82\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.092293648Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 1d39c011-f797-499f-8954-1f1d41751e5f-99d9a509-237d-43cd-9c13-86aab0a26b82 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.100636963Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"applying SMF actions for region 1d39c011-f797-499f-8954-1f1d41751e5f running snapshot 99d9a509-237d-43cd-9c13-86aab0a26b82 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.10479749Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.104841091Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"uri":"/crucible/0/regions/1d39c011-f797-499f-8954-1f1d41751e5f/snapshots/99d9a509-237d-43cd-9c13-86aab0a26b82/run","method":"POST","req_id":"bdd6c4cd-7ef7-4ba6-9d33-5236b8adc43a","remote_addr":"[fd00:1122:3344:101::a]:44881","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":29994,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-1d39c011-f797-499f-8954-1f1d41751e5f-99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.114297226Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-1d39c011-f797-499f-8954-1f1d41751e5f-99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.12419064Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-1d39c011-f797-499f-8954-1f1d41751e5f-99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.157502298Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1d39c011-f797-499f-8954-1f1d41751e5f/.zfs/snapshot/99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.157775293Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.157834594Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.157880975Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.157920896Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.157970957Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.169078073Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.181126346Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"worker"}
{"msg":"region 1d39c011-f797-499f-8954-1f1d41751e5f running snapshot 99d9a509-237d-43cd-9c13-86aab0a26b82 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.181143647Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"path is \"/data/regions/1d39c011-f797-499f-8954-1f1d41751e5f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.303932445Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"dataset is oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.312394962Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"datafile"}
{"msg":"snapshot is oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f@99d9a509-237d-43cd-9c13-86aab0a26b82","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.320289128Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4ad5e09d-9224-40f8-82f3-d956995ddd99/crucible/regions/1d39c011-f797-499f-8954-1f1d41751e5f\", \"99d9a509-237d-43cd-9c13-86aab0a26b82\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.320321569Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T00:58:17.328199855Z","hostname":"oxz_crucible_e3773db0-2640-4087-964f-b0d35e1f81ed","pid":26538,"uri":"/crucible/0/regions/1d39c011-f797-499f-8954-1f1d41751e5f/snapshots/99d9a509-237d-43cd-9c13-86aab0a26b82/run","method":"POST","req_id":"445e82b5-74a0-4275-845d-6a4cfb6087f5","remote_addr":"[fd00:1122:3344:101::a]:44881","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24293,"response_code":"200"}