[ Dec 20 21:12:01 Enabled. ]
[ Dec 20 21:12:01 Rereading configuration. ]
[ Dec 20 21:12:11 Rereading configuration. ]
[ Dec 20 21:12:18 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_4d0b1b25-4a74-46a6-945f-18014df947c1/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_4d0b1b25-4a74-46a6-945f-18014df947c1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:12:18.512577483Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:12:18.515913813Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:12:18.515925274Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:12:18.529023188Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:12:18.645577466Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.152780139Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35747"}
{"msg":"region a931657d-a0b5-4999-8579-6b31de21160e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.166047909Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.168358756Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.173424274Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"uri":"/crucible/0/regions","method":"POST","req_id":"ebac88cf-0ae2-4d15-b054-7b2875247a3f","remote_addr":"[fd00:1122:3344:101::a]:35747","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":11455,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_4d0b1b25-4a74-46a6-945f-18014df947c1/crucible/regions/a931657d-a0b5-4999-8579-6b31de21160e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.180678865Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_4d0b1b25-4a74-46a6-945f-18014df947c1/crucible/regions/a931657d-a0b5-4999-8579-6b31de21160e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.180700045Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a931657d-a0b5-4999-8579-6b31de21160e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a931657d-a0b5-4999-8579-6b31de21160e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.239172775Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"region":"a931657d-a0b5-4999-8579-6b31de21160e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a931657d-a0b5-4999-8579-6b31de21160e\" \"--data\" \"/data/regions/a931657d-a0b5-4999-8579-6b31de21160e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.241348998Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"region":"a931657d-a0b5-4999-8579-6b31de21160e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.364681151Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"region":"a931657d-a0b5-4999-8579-6b31de21160e","component":"worker"}
{"msg":"region a931657d-a0b5-4999-8579-6b31de21160e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.364748213Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.364811015Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a931657d-a0b5-4999-8579-6b31de21160e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.365229689Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a931657d-a0b5-4999-8579-6b31de21160e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.370119801Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.382561554Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a931657d-a0b5-4999-8579-6b31de21160e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.387137836Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a931657d-a0b5-4999-8579-6b31de21160e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.38788079Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.387950452Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.388000584Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.388040915Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.39300266Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.397182689Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:06.509922959Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"uri":"/crucible/0/regions","method":"POST","req_id":"1bed9dc3-e4f4-4ddb-902d-59fd5d9a311c","remote_addr":"[fd00:1122:3344:101::a]:35747","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.168623298Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"uri":"/crucible/0/regions/a931657d-a0b5-4999-8579-6b31de21160e","method":"GET","req_id":"361e0cf6-aaf0-4d65-9db8-de603eefdeac","remote_addr":"[fd00:1122:3344:101::a]:35747","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"path is \"/data/regions/a931657d-a0b5-4999-8579-6b31de21160e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.172341581Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"datafile"}
{"msg":"dataset is oxp_4d0b1b25-4a74-46a6-945f-18014df947c1/crucible/regions/a931657d-a0b5-4999-8579-6b31de21160e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.183189601Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"datafile"}
{"msg":"region a931657d-a0b5-4999-8579-6b31de21160e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.193047179Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.193212914Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.193269866Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"uri":"/crucible/0/regions/a931657d-a0b5-4999-8579-6b31de21160e","method":"DELETE","req_id":"9c9fcbfb-1165-4813-9883-988fda339ceb","remote_addr":"[fd00:1122:3344:101::a]:35747","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":20950,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-a931657d-a0b5-4999-8579-6b31de21160e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.194314481Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.196146631Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"uri":"/crucible/0/regions/a931657d-a0b5-4999-8579-6b31de21160e","method":"GET","req_id":"34a9e834-f736-4f81-ad95-991524f2cf49","remote_addr":"[fd00:1122:3344:101::a]:35747","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":34,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.199095389Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_4d0b1b25-4a74-46a6-945f-18014df947c1/crucible/regions/a931657d-a0b5-4999-8579-6b31de21160e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.210594521Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"region":"a931657d-a0b5-4999-8579-6b31de21160e","component":"worker"}
{"msg":"region a931657d-a0b5-4999-8579-6b31de21160e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.268323406Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T21:13:07.300426692Z","hostname":"oxz_crucible_f8384d9e-d33e-4c2e-a1d2-93392d74ac89","pid":22809,"uri":"/crucible/0/regions/a931657d-a0b5-4999-8579-6b31de21160e","method":"GET","req_id":"94ee7ae9-0660-4ed7-82ad-418e28e7506c","remote_addr":"[fd00:1122:3344:101::a]:35747","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":81,"response_code":"200"}