[ Dec  6 01:43:03 Disabled. ]
[ Dec  6 01:43:03 Rereading configuration. ]
[ Dec  6 01:43:07 Rereading configuration. ]
[ Dec  6 01:43:08 Rereading configuration. ]
[ Dec  6 01:43:08 Enabled. ]
[ Dec  6 01:43:08 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address [fd00:1122:3344:1::1]:5353 --dns-address [fd00:1122:3344:1::1]:53 &"). ]
[ Dec  6 01:43:08 Method "start" exited with status 0. ]
Config { log: File { level: Info, path: "/dev/stdout", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, request_body_max_bytes: 104857600, default_handler_task_mode: Detached }, storage: Config { storage_path: "/data/dns", keep_old_generations: 3 } }
note: configured to log to "/dev/stdout"
{"msg":"config","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:08.127484718Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"dns_config":"Config { bind_address: [fd00:1122:3344:1::1]:53 }","config":"Config { log: File { level: Info, path: \"/dev/stdout\", if_exists: Append }, dropshot: ConfigDropshot { bind_address: [fd00:1122:3344:1::1]:5353, request_body_max_bytes: 104857600, default_handler_task_mode: Detached }, storage: Config { storage_path: \"/data/dns\", keep_old_generations: 3 } }"}
{"msg":"opening sled database","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:08.127705352Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"component":"store","path":"/data/dns"}
{"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:08.146486758Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"component":"store"}
{"msg":"pruning trees for generations older than 0","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:08.146500448Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"component":"store","keep":3}
{"msg":"DNS server bound to address","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:08.146538809Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"component":"dns","local_address":"[fd00:1122:3344:1::1]:53"}
{"msg":"listening","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:08.146928118Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http"}
{"msg":"accepted connection","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:09.392486854Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","remote_addr":"[fd00:1122:3344:1::2]:49592"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:43:09.39321453Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::1]:54705","req_id":"a1fd3b34-890a-4f05-b06a-0f216c1851fa","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"_nexus._tcp.control-plane.oxide.internal.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:43:09.39323244Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:1::2]:54833","req_id":"06f051f0-a8db-4eac-a380-656cb8f73249","component":"dns"}
{"msg":"attempting generation update","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:09.393793702Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"new_generation":1,"req_id":"6b76c4ee-76fc-4963-b52b-0216ea88d810","component":"store"}
{"msg":"pruning trees for generations newer than 0","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:09.393810913Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"component":"store"}
{"msg":"pruning trees for generations older than 1","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:09.40389164Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"component":"store","keep":3}
{"msg":"updated generation","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:09.403904001Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"new_generation":1,"req_id":"6b76c4ee-76fc-4963-b52b-0216ea88d810","component":"store"}
{"msg":"request completed","v":0,"name":"dns-server","level":30,"time":"2023-12-06T01:43:09.403922711Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"uri":"/config","method":"PUT","req_id":"6b76c4ee-76fc-4963-b52b-0216ea88d810","remote_addr":"[fd00:1122:3344:1::2]:49592","local_addr":"[fd00:1122:3344:1::1]:5353","component":"http","latency_us":10500,"response_code":"204"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_176b7585-5339-4913-a528-c6894d494cd8.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.931042922Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::7]:63937","req_id":"dcc78f94-d17e-45af-8881-46e6aacfa024","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_176b7585-5339-4913-a528-c6894d494cd8.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.931474451Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::7]:40252","req_id":"86b9b1c2-c0bc-40a6-87e6-839c656f0c57","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_176b7585-5339-4913-a528-c6894d494cd8.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.932200217Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::7]:39241","req_id":"a1fd69fd-7efe-421d-9351-d099917d465e","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_176b7585-5339-4913-a528-c6894d494cd8.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.932429472Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::7]:62050","req_id":"b74d2c54-85fa-4f95-9bba-ecd998dc6ba9","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_176b7585-5339-4913-a528-c6894d494cd8.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.933481155Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::7]:45907","req_id":"4414b317-da7d-4a99-9a36-abe218dcedcc","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_176b7585-5339-4913-a528-c6894d494cd8.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.933767681Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::7]:51211","req_id":"787da55b-6aff-4478-871b-0febe78ba9f3","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c8a703de-60c8-4e01-8cc5-d48dce4df393.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.956938501Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::5]:60821","req_id":"43f17d90-602f-47a3-9407-db6d98427288","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c8a703de-60c8-4e01-8cc5-d48dce4df393.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.957235117Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::5]:57583","req_id":"6287c754-3264-422c-940a-aa1ece716a7f","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c8a703de-60c8-4e01-8cc5-d48dce4df393.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.958126267Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::5]:46074","req_id":"49312e7b-4939-4b43-aac0-051902b5cca9","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c8a703de-60c8-4e01-8cc5-d48dce4df393.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.958395333Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::5]:48855","req_id":"aedebe8e-c29e-4399-a22d-d18d4e9e33b1","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c8a703de-60c8-4e01-8cc5-d48dce4df393.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.959620419Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::5]:52890","req_id":"19957d9b-ba14-46fb-b41e-bc7f662fe54f","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_c8a703de-60c8-4e01-8cc5-d48dce4df393.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:00.959909865Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::5]:61098","req_id":"7629efd7-8fc7-43f0-b7e1-ee4f34f29252","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_589bb911-b0c9-45f2-9736-fa1eb9b3b271.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.049424788Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::6]:47090","req_id":"895181a4-76b5-445e-a02a-7ed55bbda509","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_589bb911-b0c9-45f2-9736-fa1eb9b3b271.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.049725254Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::6]:59329","req_id":"765e665f-4a9f-49b6-bb9d-a8a3b994a27d","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_589bb911-b0c9-45f2-9736-fa1eb9b3b271.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.050404949Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::6]:62734","req_id":"47f702cb-5328-4338-b759-7ae7120133a5","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_589bb911-b0c9-45f2-9736-fa1eb9b3b271.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.051242497Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::6]:38405","req_id":"85e5a355-85f3-44ba-b534-5aeb3e94bfb3","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_589bb911-b0c9-45f2-9736-fa1eb9b3b271.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.051655746Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::6]:39372","req_id":"3852cd69-6346-4e7e-b832-23c6d983fb1f","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_589bb911-b0c9-45f2-9736-fa1eb9b3b271.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.052463163Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::6]:37158","req_id":"ab63d964-56dd-4ea9-93e6-ed5dc85c8947","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_bc420a00-86c4-4c4f-b6b9-be2069fb1960.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.126476051Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::3]:39020","req_id":"1c543465-d72b-4fdc-ac33-bf1a42b22e78","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_bc420a00-86c4-4c4f-b6b9-be2069fb1960.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.127490213Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::3]:39047","req_id":"1b6ce6af-92e9-472f-af7a-2254eef54f4a","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_bc420a00-86c4-4c4f-b6b9-be2069fb1960.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.12779447Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::3]:61591","req_id":"e3bbadb0-b700-4fee-a90f-60f34025a7aa","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_bc420a00-86c4-4c4f-b6b9-be2069fb1960.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.128654269Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::3]:36667","req_id":"01fb1815-8d14-46a3-a9ed-d8283100069e","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_bc420a00-86c4-4c4f-b6b9-be2069fb1960.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.12962759Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::3]:40658","req_id":"32735fbe-7af4-4aab-86f5-54a304af86df","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_bc420a00-86c4-4c4f-b6b9-be2069fb1960.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.129874835Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::3]:50969","req_id":"910d3c43-51b0-410e-8d9a-6fb16debbf76","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4670e6d0-1b8d-40f9-879c-d59c3cfdd99f.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.183602485Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::4]:33940","req_id":"b9d7f2f6-ef48-4b35-8c44-b19a051bd8c3","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4670e6d0-1b8d-40f9-879c-d59c3cfdd99f.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.183916712Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::4]:43189","req_id":"f916f0ad-c065-4429-bf4e-15d262ea0a5b","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4670e6d0-1b8d-40f9-879c-d59c3cfdd99f.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.184712389Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::4]:48627","req_id":"9d09b7ca-000e-45ed-b37d-03d617f4d156","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4670e6d0-1b8d-40f9-879c-d59c3cfdd99f.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.184973815Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::4]:48244","req_id":"51642efe-e89e-4e35-9b11-de05d62cfd03","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4670e6d0-1b8d-40f9-879c-d59c3cfdd99f.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.185980867Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::4]:44514","req_id":"e980535b-5950-4673-8e5e-64813cd2b981","component":"dns"}
{"msg":"failed to handle incoming DNS message: SERVFAIL: server is not authoritative for name: \"oxz_cockroachdb_4670e6d0-1b8d-40f9-879c-d59c3cfdd99f.\"","v":0,"name":"dns-server","level":50,"time":"2023-12-06T01:45:01.186833755Z","hostname":"oxz_internal_dns_69a29de5-89ac-4785-b2ed-5aeea391c4bc","pid":4757,"peer_addr":"[fd00:1122:3344:101::4]:41246","req_id":"0f20ef93-5a30-449e-9d20-62eb3fb94bfc","component":"dns"}