[ Oct 30 19:45:48 Enabled. ]
[ Oct 30 19:45:48 Rereading configuration. ]
[ Oct 30 19:45:52 Rereading configuration. ]
[ Oct 30 19:45:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ab4adf44-36fd-4213-adae-9e4abc19a56b/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ab4adf44-36fd-4213-adae-9e4abc19a56b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:45:54.77870323Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:45:54.782762858Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:45:54.782776618Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:45:54.797789178Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:45:54.927707301Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.736763983Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49460"}
{"msg":"region a19a4ca3-085b-4bc9-bd02-ce26fc19a307 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.775474635Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.78544645Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"uri":"/crucible/0/regions","method":"POST","req_id":"a054fcf4-c293-4c3f-b47d-d82601d65e82","remote_addr":"[fd00:1122:3344:101::a]:49460","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19286,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.785476461Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_ab4adf44-36fd-4213-adae-9e4abc19a56b/crucible/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.794602725Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_ab4adf44-36fd-4213-adae-9e4abc19a56b/crucible/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.794642775Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a19a4ca3-085b-4bc9-bd02-ce26fc19a307\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.849157788Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"region":"a19a4ca3-085b-4bc9-bd02-ce26fc19a307","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a19a4ca3-085b-4bc9-bd02-ce26fc19a307\" \"--data\" \"/data/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:50.853149975Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"region":"a19a4ca3-085b-4bc9-bd02-ce26fc19a307","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.106297941Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"uri":"/crucible/0/regions","method":"POST","req_id":"84e6ebd3-187e-4c6b-b5f7-7b07415c8bbb","remote_addr":"[fd00:1122:3344:101::a]:49460","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.142048464Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"region":"a19a4ca3-085b-4bc9-bd02-ce26fc19a307","component":"worker"}
{"msg":"region a19a4ca3-085b-4bc9-bd02-ce26fc19a307 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.142087935Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.142208558Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a19a4ca3-085b-4bc9-bd02-ce26fc19a307","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.144670814Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a19a4ca3-085b-4bc9-bd02-ce26fc19a307","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.148740721Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.165734654Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a19a4ca3-085b-4bc9-bd02-ce26fc19a307","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.168674273Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.176115931Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.176188323Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.176239414Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.176297706Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.180073956Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.184412402Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:51.570793538Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"uri":"/crucible/0/regions","method":"POST","req_id":"bcdf01f8-67d3-403c-94be-b9b760742673","remote_addr":"[fd00:1122:3344:101::a]:49460","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.029831811Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"uri":"/crucible/0/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307","method":"GET","req_id":"21286075-093e-4bac-af6c-6d6c4ccbf0d3","remote_addr":"[fd00:1122:3344:101::a]:49460","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"path is \"/data/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.032280487Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"datafile"}
{"msg":"dataset is oxp_ab4adf44-36fd-4213-adae-9e4abc19a56b/crucible/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.043558987Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"datafile"}
{"msg":"region a19a4ca3-085b-4bc9-bd02-ce26fc19a307 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.054959761Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.055114865Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.055195787Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"uri":"/crucible/0/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307","method":"DELETE","req_id":"84743fd6-4436-4e00-b086-9ebb06da983d","remote_addr":"[fd00:1122:3344:101::a]:49460","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":22948,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-a19a4ca3-085b-4bc9-bd02-ce26fc19a307 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.056164843Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.058372782Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"uri":"/crucible/0/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307","method":"GET","req_id":"589d9898-f4a3-4d3b-9735-939985454ae9","remote_addr":"[fd00:1122:3344:101::a]:49460","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":30,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.06128161Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_ab4adf44-36fd-4213-adae-9e4abc19a56b/crucible/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.072804487Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"region":"a19a4ca3-085b-4bc9-bd02-ce26fc19a307","component":"worker"}
{"msg":"region a19a4ca3-085b-4bc9-bd02-ce26fc19a307 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.137042499Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-30T19:46:52.177706632Z","hostname":"oxz_crucible_b04ccba8-86b2-4354-852f-61c9fc4c0bc0","pid":26639,"uri":"/crucible/0/regions/a19a4ca3-085b-4bc9-bd02-ce26fc19a307","method":"GET","req_id":"0eb95190-1bfa-45c0-a36a-7a277fc17fb3","remote_addr":"[fd00:1122:3344:101::a]:49460","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":82,"response_code":"200"}