[ Feb 23 10:12:42 Enabled. ]
[ Feb 23 10:12:42 Rereading configuration. ]
[ Feb 23 10:12:45 Rereading configuration. ]
[ Feb 23 10:12:48 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:12:48.933205086Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:12:48.94266899Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:12:48.94268571Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:12:49.094324414Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.532421915Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:49314"}
{"msg":"region e8f91ce8-80a7-409c-b7a0-16fafc4f1584 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.563814837Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.564019052Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"uri":"/crucible/0/regions","method":"POST","req_id":"8ce5e9cb-f517-4467-b487-490f9f691818","remote_addr":"[fd00:1122:3344:101::c]:49314","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":999,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.566257327Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.575864944Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.575893864Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e8f91ce8-80a7-409c-b7a0-16fafc4f1584\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.630206216Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"region":"e8f91ce8-80a7-409c-b7a0-16fafc4f1584","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.876721462Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"uri":"/crucible/0/regions","method":"POST","req_id":"62a036cf-3ca2-4c59-8fba-d9fcf1ca8fbd","remote_addr":"[fd00:1122:3344:101::c]:49314","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.886207456Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"region":"e8f91ce8-80a7-409c-b7a0-16fafc4f1584","component":"worker"}
{"msg":"region e8f91ce8-80a7-409c-b7a0-16fafc4f1584 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.886235117Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.88638655Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e8f91ce8-80a7-409c-b7a0-16fafc4f1584","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.88687791Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e8f91ce8-80a7-409c-b7a0-16fafc4f1584","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.891612467Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.901390027Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e8f91ce8-80a7-409c-b7a0-16fafc4f1584","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.904640724Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.90542273Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.905565263Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.905615284Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.905658445Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.910430772Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:38.91472427Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.293435952Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"uri":"/crucible/0/regions","method":"POST","req_id":"18ecd6c2-500a-488c-a5bc-7abf223605f2","remote_addr":"[fd00:1122:3344:101::c]:49314","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.706235812Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:51618"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.706416195Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"uri":"/crucible/0/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584","method":"GET","req_id":"f6970834-a60b-4568-a4de-609d541df80d","remote_addr":"[fd00:1122:3344:101::c]:51618","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":158,"response_code":"200"}
{"msg":"path is \"/data/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.706575779Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"datafile"}
{"msg":"dataset is oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.71543065Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"datafile"}
{"msg":"region e8f91ce8-80a7-409c-b7a0-16fafc4f1584 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.723406463Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.723575816Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.723650618Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"uri":"/crucible/0/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584","method":"DELETE","req_id":"c949fad2-f1b2-4396-be60-5b10584604c1","remote_addr":"[fd00:1122:3344:101::c]:51618","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":17093,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.723885543Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"uri":"/crucible/0/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584","method":"GET","req_id":"8037a9b3-c34d-4c8c-86cb-a65e6479ab10","remote_addr":"[fd00:1122:3344:101::c]:51618","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-e8f91ce8-80a7-409c-b7a0-16fafc4f1584 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.724591237Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.728557118Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.737951041Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"region":"e8f91ce8-80a7-409c-b7a0-16fafc4f1584","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.777707895Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"uri":"/crucible/0/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584","method":"GET","req_id":"5a54ed08-c646-4443-9cda-b858b4659c84","remote_addr":"[fd00:1122:3344:101::c]:51618","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":552,"response_code":"200"}
{"msg":"region e8f91ce8-80a7-409c-b7a0-16fafc4f1584 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.791957506Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-23T10:13:39.94699144Z","hostname":"oxz_crucible_73112093-6328-4876-b922-117923d9e552","pid":24139,"uri":"/crucible/0/regions/e8f91ce8-80a7-409c-b7a0-16fafc4f1584","method":"GET","req_id":"2e5fc89f-2893-45b8-b76d-30d12a617b82","remote_addr":"[fd00:1122:3344:101::c]:51618","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":36,"response_code":"200"}