[ Mar 21 18:03:35 Enabled. ]
[ Mar 21 18:03:35 Rereading configuration. ]
[ Mar 21 18:03:36 Rereading configuration. ]
[ Mar 21 18:03:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:38.963480233Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:38.966911664Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:38.966935115Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:03:39.228637225Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.863689572Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56290"}
{"msg":"region 13f2b000-e4cf-4031-a10a-25b6e6328983 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.941414785Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.944632432Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"uri":"/crucible/0/regions","method":"POST","req_id":"41f52572-f704-403b-8188-68fbc4f4ccd1","remote_addr":"[fd00:1122:3344:101::a]:56290","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19844,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.947072966Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/13f2b000-e4cf-4031-a10a-25b6e6328983","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.95790144Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/13f2b000-e4cf-4031-a10a-25b6e6328983","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:05.957940461Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"13f2b000-e4cf-4031-a10a-25b6e6328983\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/13f2b000-e4cf-4031-a10a-25b6e6328983\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.01367004Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"region":"13f2b000-e4cf-4031-a10a-25b6e6328983","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.13531499Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"uri":"/crucible/0/regions","method":"POST","req_id":"8a3f71c4-7807-409d-96a3-7484cc291689","remote_addr":"[fd00:1122:3344:101::a]:56290","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.320532969Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"region":"13f2b000-e4cf-4031-a10a-25b6e6328983","component":"worker"}
{"msg":"region 13f2b000-e4cf-4031-a10a-25b6e6328983 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.32056843Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.320727483Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-13f2b000-e4cf-4031-a10a-25b6e6328983","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.321311283Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-13f2b000-e4cf-4031-a10a-25b6e6328983","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.326096209Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.342948231Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-13f2b000-e4cf-4031-a10a-25b6e6328983","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.345931124Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/13f2b000-e4cf-4031-a10a-25b6e6328983","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.347294059Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.349726552Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.349782864Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.349823164Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.36240625Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.369070049Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:06.876601955Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"uri":"/crucible/0/regions","method":"POST","req_id":"e5b6c458-b53d-400b-91fb-90ea645891c8","remote_addr":"[fd00:1122:3344:101::a]:56290","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":108,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.608239417Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53037"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.608323319Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"uri":"/crucible/0/regions/13f2b000-e4cf-4031-a10a-25b6e6328983","method":"GET","req_id":"d603ac07-10cc-4a9e-bc31-67096ca7bef0","remote_addr":"[fd00:1122:3344:101::a]:53037","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"path is \"/data/regions/13f2b000-e4cf-4031-a10a-25b6e6328983\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.608535172Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"datafile"}
{"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/13f2b000-e4cf-4031-a10a-25b6e6328983","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.619283855Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"datafile"}
{"msg":"region 13f2b000-e4cf-4031-a10a-25b6e6328983 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.628870397Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.629121081Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.629179913Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"uri":"/crucible/0/regions/13f2b000-e4cf-4031-a10a-25b6e6328983","method":"DELETE","req_id":"62616878-046c-48aa-bb1b-1dd1244dc38d","remote_addr":"[fd00:1122:3344:101::a]:53037","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20680,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.629352176Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"uri":"/crucible/0/regions/13f2b000-e4cf-4031-a10a-25b6e6328983","method":"GET","req_id":"53fdc033-54f2-46fe-9561-c6547845c676","remote_addr":"[fd00:1122:3344:101::a]:53037","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-13f2b000-e4cf-4031-a10a-25b6e6328983 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.630385894Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.634885115Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/13f2b000-e4cf-4031-a10a-25b6e6328983\"","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.644615309Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"region":"13f2b000-e4cf-4031-a10a-25b6e6328983","component":"worker"}
{"msg":"region 13f2b000-e4cf-4031-a10a-25b6e6328983 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.689769659Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-03-21T18:10:29.744317546Z","hostname":"oxz_crucible_56a96eb3-dee7-41e6-a406-5830afb2b41d","pid":20639,"uri":"/crucible/0/regions/13f2b000-e4cf-4031-a10a-25b6e6328983","method":"GET","req_id":"df35cf33-2481-4a33-912a-807d08e9fa3b","remote_addr":"[fd00:1122:3344:101::a]:53037","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":69,"response_code":"200"}