[ Jun  4 01:18:05 Enabled. ]
[ Jun  4 01:18:05 Rereading configuration. ]
[ Jun  4 01:18:09 Rereading configuration. ]
[ Jun  4 01:18:18 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:18:18.491800615Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:18:18.495013255Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:18:18.495021885Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:18:18.511501928Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:18:18.596893265Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.26509941Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:32814"}
{"msg":"region e86e10ff-c930-4473-bf14-1928afbdca30 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.298371054Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.303086196Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"uri":"/crucible/0/regions","method":"POST","req_id":"4600c94c-eb95-4c81-9c0f-6d9fcefc36e8","remote_addr":"[fd00:1122:3344:101::c]:32814","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20866,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.303701443Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.313436006Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.313499078Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e86e10ff-c930-4473-bf14-1928afbdca30\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e86e10ff-c930-4473-bf14-1928afbdca30\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.407348913Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"region":"e86e10ff-c930-4473-bf14-1928afbdca30","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e86e10ff-c930-4473-bf14-1928afbdca30\" \"--data\" \"/data/regions/e86e10ff-c930-4473-bf14-1928afbdca30\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.409507073Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"region":"e86e10ff-c930-4473-bf14-1928afbdca30","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.442555781Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"uri":"/crucible/0/regions","method":"POST","req_id":"183d1bbe-70dd-47dd-a9c3-a56bda7fd80f","remote_addr":"[fd00:1122:3344:101::c]:32814","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.581043598Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"region":"e86e10ff-c930-4473-bf14-1928afbdca30","component":"worker"}
{"msg":"region e86e10ff-c930-4473-bf14-1928afbdca30 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.581132901Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.581352507Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e86e10ff-c930-4473-bf14-1928afbdca30","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.581757858Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e86e10ff-c930-4473-bf14-1928afbdca30","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.586557553Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.6024871Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e86e10ff-c930-4473-bf14-1928afbdca30","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.605712131Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e86e10ff-c930-4473-bf14-1928afbdca30","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.606281107Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.606342069Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.60637893Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.606423581Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.611432851Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.626454863Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:19:02.749758714Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"uri":"/crucible/0/regions","method":"POST","req_id":"f3a34bc1-1a4e-4720-8379-580f2506f3d8","remote_addr":"[fd00:1122:3344:101::c]:32814","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.301340164Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:46834"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.305224803Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"uri":"/crucible/0/regions/e86e10ff-c930-4473-bf14-1928afbdca30","method":"GET","req_id":"ac777f08-9b5c-4384-ac6d-57c1d65a50bb","remote_addr":"[fd00:1122:3344:101::c]:46834","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":3825,"response_code":"200"}
{"msg":"path is \"/data/regions/e86e10ff-c930-4473-bf14-1928afbdca30\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.308541436Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"dataset is oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.319298158Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"snapshot is oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30@1e88d601-271a-429c-80c9-3d1b7e327115","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.328406634Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30\", \"1e88d601-271a-429c-80c9-3d1b7e327115\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.328441855Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.340426581Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"uri":"/crucible/0/regions/e86e10ff-c930-4473-bf14-1928afbdca30/snapshots/1e88d601-271a-429c-80c9-3d1b7e327115","method":"GET","req_id":"ff1e7de5-6e16-4881-846a-9d48a4f85cc4","remote_addr":"[fd00:1122:3344:101::c]:46834","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":31878,"response_code":"200"}
{"msg":"path is \"/data/regions/e86e10ff-c930-4473-bf14-1928afbdca30\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.344098684Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"dataset is oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.352057007Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"snapshot is oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30@1e88d601-271a-429c-80c9-3d1b7e327115","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.359521317Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30\", \"1e88d601-271a-429c-80c9-3d1b7e327115\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.359549658Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot e86e10ff-c930-4473-bf14-1928afbdca30-1e88d601-271a-429c-80c9-3d1b7e327115 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.369722223Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"applying SMF actions for region e86e10ff-c930-4473-bf14-1928afbdca30 running snapshot 1e88d601-271a-429c-80c9-3d1b7e327115 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.370349241Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.370369492Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"uri":"/crucible/0/regions/e86e10ff-c930-4473-bf14-1928afbdca30/snapshots/1e88d601-271a-429c-80c9-3d1b7e327115/run","method":"POST","req_id":"333d8ca4-60e1-4516-8e27-87bded134188","remote_addr":"[fd00:1122:3344:101::c]:46834","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26308,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-e86e10ff-c930-4473-bf14-1928afbdca30-1e88d601-271a-429c-80c9-3d1b7e327115","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.386135024Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-e86e10ff-c930-4473-bf14-1928afbdca30-1e88d601-271a-429c-80c9-3d1b7e327115","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.392634217Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-e86e10ff-c930-4473-bf14-1928afbdca30-1e88d601-271a-429c-80c9-3d1b7e327115","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.404709356Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e86e10ff-c930-4473-bf14-1928afbdca30/.zfs/snapshot/1e88d601-271a-429c-80c9-3d1b7e327115","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.404976583Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.405041505Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.405091706Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.405138048Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.405199209Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.408757149Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.413249995Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"worker"}
{"msg":"region e86e10ff-c930-4473-bf14-1928afbdca30 running snapshot 1e88d601-271a-429c-80c9-3d1b7e327115 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.413271156Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"path is \"/data/regions/e86e10ff-c930-4473-bf14-1928afbdca30\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.584620396Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"dataset is oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.593060093Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"datafile"}
{"msg":"snapshot is oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30@1e88d601-271a-429c-80c9-3d1b7e327115","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.601843059Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_86d3854a-b47a-4aa2-8841-0cb5cd97c3ee/crucible/regions/e86e10ff-c930-4473-bf14-1928afbdca30\", \"1e88d601-271a-429c-80c9-3d1b7e327115\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.60188255Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T01:29:37.610216474Z","hostname":"oxz_crucible_d243cf4e-babe-4978-a234-4f731bb20f0d","pid":24253,"uri":"/crucible/0/regions/e86e10ff-c930-4473-bf14-1928afbdca30/snapshots/1e88d601-271a-429c-80c9-3d1b7e327115/run","method":"POST","req_id":"5c7c0098-29fe-42dc-8fe7-74e41cdb66c7","remote_addr":"[fd00:1122:3344:101::c]:46834","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25602,"response_code":"200"}