[ Jul 12 03:34:03 Enabled. ]
[ Jul 12 03:34:03 Rereading configuration. ]
[ Jul 12 03:34:07 Rereading configuration. ]
[ Jul 12 03:34:15 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9c393420-4edf-4d30-8594-4999e04847e6/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9c393420-4edf-4d30-8594-4999e04847e6/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:15.386195137Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:15.389477509Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:15.38949162Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:15.404555962Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:15.66446784Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.384195635Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:65457"}
{"msg":"region 7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.400557856Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.405738431Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"uri":"/crucible/0/regions","method":"POST","req_id":"c0b2633d-223b-4d2f-817d-3abb96c8eca0","remote_addr":"[fd00:1122:3344:101::b]:65457","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":11085,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.405763541Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_9c393420-4edf-4d30-8594-4999e04847e6/crucible/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.419158302Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9c393420-4edf-4d30-8594-4999e04847e6/crucible/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.419198902Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.530239718Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"region":"7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b\" \"--data\" \"/data/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.532159571Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"region":"7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.758135549Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"uri":"/crucible/0/regions","method":"POST","req_id":"73329893-c876-46ed-93e8-8b388cff96dd","remote_addr":"[fd00:1122:3344:101::b]:65457","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.759304388Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"region":"7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","component":"worker"}
{"msg":"region 7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.759322688Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.759473859Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.759954402Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.775907841Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.810079073Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.819751759Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.820647445Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.824556512Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.824616862Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.824655872Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.845671135Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:50.860719608Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.210999032Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"uri":"/crucible/0/regions","method":"POST","req_id":"00490307-2ad1-4ad6-8e0b-bfa8711bbf53","remote_addr":"[fd00:1122:3344:101::b]:65457","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":639,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.72498942Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"uri":"/crucible/0/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","method":"GET","req_id":"89ba92e9-f9b2-48b2-ba73-0640a3dc9e47","remote_addr":"[fd00:1122:3344:101::b]:65457","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":94,"response_code":"200"}
{"msg":"path is \"/data/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.727614828Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"datafile"}
{"msg":"dataset is oxp_9c393420-4edf-4d30-8594-4999e04847e6/crucible/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.73665952Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"datafile"}
{"msg":"region 7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.746226434Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.746392316Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.746464016Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"uri":"/crucible/0/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","method":"DELETE","req_id":"ed2b46b1-6705-48ed-9d7b-2f304ceba561","remote_addr":"[fd00:1122:3344:101::b]:65457","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":18922,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.747636574Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.749035264Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"uri":"/crucible/0/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","method":"GET","req_id":"d8f234a1-bbd5-4c38-970a-df93cd1f4a9f","remote_addr":"[fd00:1122:3344:101::b]:65457","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":23,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.755026954Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9c393420-4edf-4d30-8594-4999e04847e6/crucible/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.763866005Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"region":"7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","component":"worker"}
{"msg":"region 7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.838951056Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-12T03:34:51.8969164Z","hostname":"oxz_crucible_2cdb4901-6166-4851-a4af-cf91e1138205","pid":23223,"uri":"/crucible/0/regions/7c20ae71-9cd1-4cb9-bd15-7c4b7b5cf38b","method":"GET","req_id":"9279320c-40c5-47cd-a39e-82eec3d0c468","remote_addr":"[fd00:1122:3344:101::b]:65457","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":79,"response_code":"200"}