[ Dec 19 21:41:12 Enabled. ] [ Dec 19 21:41:12 Rereading configuration. ] [ Dec 19 21:41:14 Rereading configuration. ] [ Dec 19 21:41:20 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_595b6dc4-b331-490d-aa08-f0377a395ba7/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_595b6dc4-b331-490d-aa08-f0377a395ba7/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:20.46735326Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664} {"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:20.470695033Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:20.470705313Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:20.48420562Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:20.812871157Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:53.914040295Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:50822"} {"msg":"region 3a592643-b89e-4e65-ae25-f328140e2783 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:53.948003559Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:53.95245123Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:53.957106915Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"uri":"/crucible/0/regions","method":"POST","req_id":"548643b2-34e8-42b1-98b7-963798076d2c","remote_addr":"[fd00:1122:3344:101::a]:50822","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":14899,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_595b6dc4-b331-490d-aa08-f0377a395ba7/crucible/regions/3a592643-b89e-4e65-ae25-f328140e2783","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:53.96426652Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_595b6dc4-b331-490d-aa08-f0377a395ba7/crucible/regions/3a592643-b89e-4e65-ae25-f328140e2783","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:53.96428887Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"3a592643-b89e-4e65-ae25-f328140e2783\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3a592643-b89e-4e65-ae25-f328140e2783\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.035191726Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"region":"3a592643-b89e-4e65-ae25-f328140e2783","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3a592643-b89e-4e65-ae25-f328140e2783\" \"--data\" \"/data/regions/3a592643-b89e-4e65-ae25-f328140e2783\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.037170738Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"region":"3a592643-b89e-4e65-ae25-f328140e2783","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.172759221Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"region":"3a592643-b89e-4e65-ae25-f328140e2783","component":"worker"} {"msg":"region 3a592643-b89e-4e65-ae25-f328140e2783 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.172818402Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.172924764Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-3a592643-b89e-4e65-ae25-f328140e2783","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.173356321Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3a592643-b89e-4e65-ae25-f328140e2783","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.179256025Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.192984415Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3a592643-b89e-4e65-ae25-f328140e2783","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.195550857Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3a592643-b89e-4e65-ae25-f328140e2783","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.196178697Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.196256798Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.196302378Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.196349629Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.199670263Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.20822382Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T21:41:54.305469378Z","hostname":"oxz_crucible_f00ec4b2-88a0-4de4-aba3-586ffad5ff4b","pid":22664,"uri":"/crucible/0/regions","method":"POST","req_id":"5cea410e-5276-4889-8eea-d8d2cf8486c9","remote_addr":"[fd00:1122:3344:101::a]:50822","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":71,"response_code":"200"}