[ Jul 25 05:48:05 Enabled. ] [ Jul 25 05:48:05 Rereading configuration. ] [ Jul 25 05:48:10 Rereading configuration. ] [ Jul 25 05:48:14 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_587239d7-a145-4bb7-b24a-aff014195deb/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_587239d7-a145-4bb7-b24a-aff014195deb/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:14.152093174Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737} {"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:14.15321107Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:14.153227881Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:14.174341212Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:14.25297194Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.640659484Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:65333"} {"msg":"region e44d224d-e039-462d-be4d-65d031b64aea state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.643076412Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.648085401Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.648101281Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"uri":"/crucible/0/regions","method":"POST","req_id":"c8977743-513a-45dc-b2d5-77df4050b367","remote_addr":"[fd00:1122:3344:101::c]:65333","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":5272,"response_code":"200"} {"msg":"zfs set reservation of 1342177280 for oxp_587239d7-a145-4bb7-b24a-aff014195deb/crucible/regions/e44d224d-e039-462d-be4d-65d031b64aea","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.657476194Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_587239d7-a145-4bb7-b24a-aff014195deb/crucible/regions/e44d224d-e039-462d-be4d-65d031b64aea","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.657509315Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e44d224d-e039-462d-be4d-65d031b64aea\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e44d224d-e039-462d-be4d-65d031b64aea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.721381221Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"region":"e44d224d-e039-462d-be4d-65d031b64aea","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e44d224d-e039-462d-be4d-65d031b64aea\" \"--data\" \"/data/regions/e44d224d-e039-462d-be4d-65d031b64aea\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.721435113Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"region":"e44d224d-e039-462d-be4d-65d031b64aea","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.847076176Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"region":"e44d224d-e039-462d-be4d-65d031b64aea","component":"worker"} {"msg":"region e44d224d-e039-462d-be4d-65d031b64aea state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.847162008Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.847371883Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e44d224d-e039-462d-be4d-65d031b64aea","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.847788033Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e44d224d-e039-462d-be4d-65d031b64aea","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.851610754Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.865587316Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"uri":"/crucible/0/regions","method":"POST","req_id":"66921d2b-4ff1-4625-851b-424a57d97148","remote_addr":"[fd00:1122:3344:101::c]:65333","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":595,"response_code":"200"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.867266626Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e44d224d-e039-462d-be4d-65d031b64aea","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.873974465Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e44d224d-e039-462d-be4d-65d031b64aea","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.874715042Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.877175611Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.877228852Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.877273883Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.889483453Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:35.896787787Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.748303907Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"uri":"/crucible/0/regions/e44d224d-e039-462d-be4d-65d031b64aea","method":"GET","req_id":"6a636d68-3b9d-4631-ae11-68e1266ce30e","remote_addr":"[fd00:1122:3344:101::c]:65333","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":88,"response_code":"200"} {"msg":"path is \"/data/regions/e44d224d-e039-462d-be4d-65d031b64aea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.750581591Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"datafile"} {"msg":"dataset is oxp_587239d7-a145-4bb7-b24a-aff014195deb/crucible/regions/e44d224d-e039-462d-be4d-65d031b64aea","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.7593688Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"datafile"} {"msg":"region e44d224d-e039-462d-be4d-65d031b64aea state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.767493612Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.767634606Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.767709158Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"uri":"/crucible/0/regions/e44d224d-e039-462d-be4d-65d031b64aea","method":"DELETE","req_id":"27fec33a-cebd-4825-9c48-40ff2d90efe1","remote_addr":"[fd00:1122:3344:101::c]:65333","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":17164,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-e44d224d-e039-462d-be4d-65d031b64aea (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.768715351Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.770882173Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"uri":"/crucible/0/regions/e44d224d-e039-462d-be4d-65d031b64aea","method":"GET","req_id":"ae21ada4-2b65-45c6-b6b6-0f5f34b77e51","remote_addr":"[fd00:1122:3344:101::c]:65333","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":23,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.77288195Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_587239d7-a145-4bb7-b24a-aff014195deb/crucible/regions/e44d224d-e039-462d-be4d-65d031b64aea\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.78171265Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"region":"e44d224d-e039-462d-be4d-65d031b64aea","component":"worker"} {"msg":"region e44d224d-e039-462d-be4d-65d031b64aea state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.834508974Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:48:36.908219104Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"uri":"/crucible/0/regions/e44d224d-e039-462d-be4d-65d031b64aea","method":"GET","req_id":"ad8e5ee5-a928-41e9-934e-fc69d424b56c","remote_addr":"[fd00:1122:3344:101::c]:65333","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":710,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.874619329Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:51028"} {"msg":"region 49a81213-82c9-44d6-b7f0-551d9635997a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.875364857Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.87886311Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.878897121Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"uri":"/crucible/0/regions","method":"POST","req_id":"5356256f-907d-4af4-a8e3-3747e967a5ab","remote_addr":"[fd00:1122:3344:101::a]:51028","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":4200,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_587239d7-a145-4bb7-b24a-aff014195deb/crucible/regions/49a81213-82c9-44d6-b7f0-551d9635997a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.890042525Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_587239d7-a145-4bb7-b24a-aff014195deb/crucible/regions/49a81213-82c9-44d6-b7f0-551d9635997a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.890074456Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"49a81213-82c9-44d6-b7f0-551d9635997a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/49a81213-82c9-44d6-b7f0-551d9635997a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.998096131Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"region":"49a81213-82c9-44d6-b7f0-551d9635997a","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"49a81213-82c9-44d6-b7f0-551d9635997a\" \"--data\" \"/data/regions/49a81213-82c9-44d6-b7f0-551d9635997a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:52.999845423Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"region":"49a81213-82c9-44d6-b7f0-551d9635997a","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.127408852Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"uri":"/crucible/0/regions","method":"POST","req_id":"6edd3015-aacb-4d09-a319-a3b5c07bc77a","remote_addr":"[fd00:1122:3344:101::a]:51028","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":80,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.187033168Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"region":"49a81213-82c9-44d6-b7f0-551d9635997a","component":"worker"} {"msg":"region 49a81213-82c9-44d6-b7f0-551d9635997a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.187060299Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.187199362Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-49a81213-82c9-44d6-b7f0-551d9635997a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.188659276Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-49a81213-82c9-44d6-b7f0-551d9635997a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.200415526Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.239888823Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-49a81213-82c9-44d6-b7f0-551d9635997a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.27766792Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/49a81213-82c9-44d6-b7f0-551d9635997a","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.27809119Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.278131851Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.278178322Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.278223194Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.301023995Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.319255298Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-25T05:54:53.383573445Z","hostname":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","pid":28737,"uri":"/crucible/0/regions","method":"POST","req_id":"86dcd36e-a8dd-4602-bfc2-de56646d3b38","remote_addr":"[fd00:1122:3344:101::a]:51028","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":50,"response_code":"200"}