[ Feb 28 13:20:41 Enabled. ]
[ Feb 28 13:20:41 Rereading configuration. ]
[ Feb 28 13:20:43 Rereading configuration. ]
[ Feb 28 13:20:46 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:20:46.903853466Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:20:46.908100726Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:20:46.908112916Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:20:47.079804712Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:37.669556693Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:35730"}
{"msg":"region 26eeba9f-c774-4a8d-9c9c-ccd10587a041 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:37.749277758Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:37.754464582Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:37.754487193Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"uri":"/crucible/0/regions","method":"POST","req_id":"5d78609c-e2ab-4df1-9ab3-c27faaf2b8cc","remote_addr":"[fd00:1122:3344:101::a]:35730","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":18631,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:37.766762229Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:37.76680003Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"26eeba9f-c774-4a8d-9c9c-ccd10587a041\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:37.822139647Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"region":"26eeba9f-c774-4a8d-9c9c-ccd10587a041","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:37.959994481Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"uri":"/crucible/0/regions","method":"POST","req_id":"17db720e-1151-466a-bb38-4628b41ce273","remote_addr":"[fd00:1122:3344:101::a]:35730","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.070097689Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"region":"26eeba9f-c774-4a8d-9c9c-ccd10587a041","component":"worker"}
{"msg":"region 26eeba9f-c774-4a8d-9c9c-ccd10587a041 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.07013744Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.070231463Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-26eeba9f-c774-4a8d-9c9c-ccd10587a041","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.070650133Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-26eeba9f-c774-4a8d-9c9c-ccd10587a041","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.076412262Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.087472797Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-26eeba9f-c774-4a8d-9c9c-ccd10587a041","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.09030307Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.101502419Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.103599523Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.103661834Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.103703675Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.111788964Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.115995132Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:27:38.322097756Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"uri":"/crucible/0/regions","method":"POST","req_id":"53a8f788-7653-4d01-b59e-e1235aeb2a0a","remote_addr":"[fd00:1122:3344:101::a]:35730","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.890665911Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47562"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.893298129Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"uri":"/crucible/0/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041","method":"GET","req_id":"3a46dfdc-16ca-44f5-bcdb-9f34fe6f4125","remote_addr":"[fd00:1122:3344:101::a]:47562","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":2586,"response_code":"200"}
{"msg":"path is \"/data/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.893667099Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"datafile"}
{"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.904083217Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"datafile"}
{"msg":"region 26eeba9f-c774-4a8d-9c9c-ccd10587a041 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.912801362Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.912992597Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.913062169Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"uri":"/crucible/0/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041","method":"DELETE","req_id":"b541523f-1b83-4a91-a5f1-b10e973f91b5","remote_addr":"[fd00:1122:3344:101::a]:47562","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19453,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.913277254Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"uri":"/crucible/0/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041","method":"GET","req_id":"2ec018da-876c-414d-880b-83d0eaec679b","remote_addr":"[fd00:1122:3344:101::a]:47562","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-26eeba9f-c774-4a8d-9c9c-ccd10587a041 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.914061984Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.923526959Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.933316621Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"region":"26eeba9f-c774-4a8d-9c9c-ccd10587a041","component":"worker"}
{"msg":"region 26eeba9f-c774-4a8d-9c9c-ccd10587a041 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:17.990978357Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-28T13:28:18.031767239Z","hostname":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","pid":24230,"uri":"/crucible/0/regions/26eeba9f-c774-4a8d-9c9c-ccd10587a041","method":"GET","req_id":"bf73fbac-1b4b-417e-8107-39c3b3de7593","remote_addr":"[fd00:1122:3344:101::a]:47562","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":86,"response_code":"200"}