[ Sep  3 22:54:58 Enabled. ]
[ Sep  3 22:54:58 Rereading configuration. ]
[ Sep  3 22:54:59 Rereading configuration. ]
[ Sep  3 22:55:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_65e87123-aa5d-45b6-b440-c7eb0d292d98/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_65e87123-aa5d-45b6-b440-c7eb0d292d98/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:55:00.924534982Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:55:00.928049488Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:55:00.928065118Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:55:00.956642109Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:55:01.059897071Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.484936726Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:62090"}
{"msg":"region fb6db9a7-c5e3-492e-b165-45cbd62df010 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.514286714Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.518647689Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"uri":"/crucible/0/regions","method":"POST","req_id":"90ce8782-944e-4410-a91f-5e90185fc79f","remote_addr":"[fd00:1122:3344:101::b]:62090","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20511,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.51979845Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_65e87123-aa5d-45b6-b440-c7eb0d292d98/crucible/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.530483509Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_65e87123-aa5d-45b6-b440-c7eb0d292d98/crucible/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.53051322Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"fb6db9a7-c5e3-492e-b165-45cbd62df010\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.589081526Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"region":"fb6db9a7-c5e3-492e-b165-45cbd62df010","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"fb6db9a7-c5e3-492e-b165-45cbd62df010\" \"--data\" \"/data/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.591030056Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"region":"fb6db9a7-c5e3-492e-b165-45cbd62df010","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.702194878Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"region":"fb6db9a7-c5e3-492e-b165-45cbd62df010","component":"worker"}
{"msg":"region fb6db9a7-c5e3-492e-b165-45cbd62df010 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.702245098Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.702322529Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-fb6db9a7-c5e3-492e-b165-45cbd62df010","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.702709553Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-fb6db9a7-c5e3-492e-b165-45cbd62df010","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.710945247Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.722859498Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-fb6db9a7-c5e3-492e-b165-45cbd62df010","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.726126711Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.729102882Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.729178923Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.729221293Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.729258734Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.739985623Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.747139465Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:00.866640762Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"uri":"/crucible/0/regions","method":"POST","req_id":"30a927fe-0e29-4de6-b67a-65a279227d1d","remote_addr":"[fd00:1122:3344:101::b]:62090","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.286750249Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"uri":"/crucible/0/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010","method":"GET","req_id":"58f6300c-ee63-402e-838f-8b0729fdb815","remote_addr":"[fd00:1122:3344:101::b]:62090","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"path is \"/data/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.289212774Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"datafile"}
{"msg":"dataset is oxp_65e87123-aa5d-45b6-b440-c7eb0d292d98/crucible/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.298962354Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"datafile"}
{"msg":"region fb6db9a7-c5e3-492e-b165-45cbd62df010 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.308154187Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.308308819Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.30839649Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"uri":"/crucible/0/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010","method":"DELETE","req_id":"a262c366-5e0a-4106-947c-f611d5491dd6","remote_addr":"[fd00:1122:3344:101::b]:62090","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19211,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-fb6db9a7-c5e3-492e-b165-45cbd62df010 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.309546942Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.311824205Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"uri":"/crucible/0/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010","method":"GET","req_id":"c96b4c48-43d0-41c1-b6cc-90b1297167ae","remote_addr":"[fd00:1122:3344:101::b]:62090","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":550,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.314200819Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_65e87123-aa5d-45b6-b440-c7eb0d292d98/crucible/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.324433743Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"region":"fb6db9a7-c5e3-492e-b165-45cbd62df010","component":"worker"}
{"msg":"region fb6db9a7-c5e3-492e-b165-45cbd62df010 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.378546074Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:01.453521737Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"uri":"/crucible/0/regions/fb6db9a7-c5e3-492e-b165-45cbd62df010","method":"GET","req_id":"d5b41e5e-75b2-4bf3-af94-eee594e7dd52","remote_addr":"[fd00:1122:3344:101::b]:62090","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.019689333Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46143"}
{"msg":"region 599822e2-f573-4e31-ae08-144dacd881ec state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.019850705Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.020032827Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.020047677Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"uri":"/crucible/0/regions","method":"POST","req_id":"2101e195-9cd9-4f3f-b0bd-29bba6606203","remote_addr":"[fd00:1122:3344:101::a]:46143","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":230,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_65e87123-aa5d-45b6-b440-c7eb0d292d98/crucible/regions/599822e2-f573-4e31-ae08-144dacd881ec","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.029156849Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_65e87123-aa5d-45b6-b440-c7eb0d292d98/crucible/regions/599822e2-f573-4e31-ae08-144dacd881ec","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.02918554Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"599822e2-f573-4e31-ae08-144dacd881ec\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/599822e2-f573-4e31-ae08-144dacd881ec\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.091442864Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"region":"599822e2-f573-4e31-ae08-144dacd881ec","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"599822e2-f573-4e31-ae08-144dacd881ec\" \"--data\" \"/data/regions/599822e2-f573-4e31-ae08-144dacd881ec\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.091489114Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"region":"599822e2-f573-4e31-ae08-144dacd881ec","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.169875413Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"region":"599822e2-f573-4e31-ae08-144dacd881ec","component":"worker"}
{"msg":"region 599822e2-f573-4e31-ae08-144dacd881ec state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.169914053Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.169994454Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-599822e2-f573-4e31-ae08-144dacd881ec","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.171366588Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-599822e2-f573-4e31-ae08-144dacd881ec","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.17459187Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.184027657Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-599822e2-f573-4e31-ae08-144dacd881ec","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.191770545Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/599822e2-f573-4e31-ae08-144dacd881ec","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.192004158Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.192059368Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.192100849Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.192139519Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.195546654Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.20100853Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-03T22:56:26.259195612Z","hostname":"oxz_crucible_45070961-6783-4e7d-a949-9460d17dab2c","pid":20909,"uri":"/crucible/0/regions","method":"POST","req_id":"23603905-bde6-4016-9042-30808d6780cc","remote_addr":"[fd00:1122:3344:101::a]:46143","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":69,"response_code":"200"}