{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:01:58.915684914Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55086"} {"msg":"region 5bd48c12-abe2-4dfa-87a7-2f2885e56371 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:01:58.98980185Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:01:59.003053593Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:01:59.008081799Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions","method":"POST","req_id":"edd3c4a7-f702-4aa1-a59e-e3cb97a7946e","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":49899,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_ec7fe47f-2f1c-4e69-8b17-4e672bac9e19/crucible/regions/5bd48c12-abe2-4dfa-87a7-2f2885e56371","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:01:59.041324894Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_ec7fe47f-2f1c-4e69-8b17-4e672bac9e19/crucible/regions/5bd48c12-abe2-4dfa-87a7-2f2885e56371","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:01:59.041342344Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:01:59.146389322Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions","method":"POST","req_id":"0cec8662-38fb-4780-9550-7a18494e69c9","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":77,"response_code":"200"} {"msg":"creating region Region { id: RegionId(\"5bd48c12-abe2-4dfa-87a7-2f2885e56371\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5bd48c12-abe2-4dfa-87a7-2f2885e56371\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:01:59.35773426Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"region":"5bd48c12-abe2-4dfa-87a7-2f2885e56371","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5bd48c12-abe2-4dfa-87a7-2f2885e56371\" \"--data\" \"/data/regions/5bd48c12-abe2-4dfa-87a7-2f2885e56371\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:01:59.360019333Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"region":"5bd48c12-abe2-4dfa-87a7-2f2885e56371","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:01:59.499266134Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions","method":"POST","req_id":"877e1975-6c82-4e9e-b99e-595e5c7ecc70","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":78,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:00.464007067Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions","method":"POST","req_id":"964dea70-9ac2-4509-81b9-cd7bacd4f37d","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":58,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:00.855081209Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"region":"5bd48c12-abe2-4dfa-87a7-2f2885e56371","component":"worker"} {"msg":"region 5bd48c12-abe2-4dfa-87a7-2f2885e56371 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:00.85512724Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:00.855289283Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-5bd48c12-abe2-4dfa-87a7-2f2885e56371","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:00.855893094Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5bd48c12-abe2-4dfa-87a7-2f2885e56371","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:01.043849926Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:01.346918176Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5bd48c12-abe2-4dfa-87a7-2f2885e56371","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:01.429444693Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5bd48c12-abe2-4dfa-87a7-2f2885e56371","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:01.430242099Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:01.43031572Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:01.430364261Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:01.430404942Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:01.453759268Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:01.478250086Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:03.305353416Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions","method":"POST","req_id":"318aa770-0613-4cc8-9ee5-06b726d4d2c9","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":64,"response_code":"200"} {"msg":"region 7b444c22-30e2-4e1d-8233-8c7a5591282e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.036252765Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.036411198Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.036428008Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions","method":"POST","req_id":"76504533-e6a2-4cea-8752-a67baba439fd","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":214,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_ec7fe47f-2f1c-4e69-8b17-4e672bac9e19/crucible/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.045858978Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_ec7fe47f-2f1c-4e69-8b17-4e672bac9e19/crucible/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.045892469Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"7b444c22-30e2-4e1d-8233-8c7a5591282e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.112860109Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"region":"7b444c22-30e2-4e1d-8233-8c7a5591282e","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7b444c22-30e2-4e1d-8233-8c7a5591282e\" \"--data\" \"/data/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.11292089Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"region":"7b444c22-30e2-4e1d-8233-8c7a5591282e","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.216614121Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions","method":"POST","req_id":"1669ee3f-7bbe-4946-b7ec-20c712806df9","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":83,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.298752501Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"region":"7b444c22-30e2-4e1d-8233-8c7a5591282e","component":"worker"} {"msg":"region 7b444c22-30e2-4e1d-8233-8c7a5591282e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.298778641Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.298890993Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-7b444c22-30e2-4e1d-8233-8c7a5591282e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.303779777Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7b444c22-30e2-4e1d-8233-8c7a5591282e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.314094714Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.32958872Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7b444c22-30e2-4e1d-8233-8c7a5591282e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.3384736Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.338706174Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.338753675Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.338794006Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.338832536Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.347270888Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.357056924Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:09.885047653Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions","method":"POST","req_id":"371d938f-8c8b-4fef-a08b-1ac2c69e2e7a","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":50,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.529025795Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e","method":"GET","req_id":"ebb70636-9267-41f1-bfad-76df5c9ffc98","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":639,"response_code":"200"} {"msg":"path is \"/data/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.531991301Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"datafile"} {"msg":"dataset is oxp_ec7fe47f-2f1c-4e69-8b17-4e672bac9e19/crucible/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.542591414Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"datafile"} {"msg":"region 7b444c22-30e2-4e1d-8233-8c7a5591282e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.552908761Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.553101555Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.553172406Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e","method":"DELETE","req_id":"1a232db4-a29e-4d51-91f4-c336f46eb7df","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21210,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-7b444c22-30e2-4e1d-8233-8c7a5591282e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.554347788Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.556550481Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e","method":"GET","req_id":"30244b8c-096d-4750-8fcb-bcaa1b571b01","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":29,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.559206881Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_ec7fe47f-2f1c-4e69-8b17-4e672bac9e19/crucible/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.568852096Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"region":"7b444c22-30e2-4e1d-8233-8c7a5591282e","component":"worker"} {"msg":"region 7b444c22-30e2-4e1d-8233-8c7a5591282e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.629320031Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-16T21:02:35.704692171Z","hostname":"oxz_crucible_3cf24053-4af1-4d9e-a59a-af36b925a98f","pid":21649,"uri":"/crucible/0/regions/7b444c22-30e2-4e1d-8233-8c7a5591282e","method":"GET","req_id":"5a359e23-967c-4531-9299-9e715be3e46e","remote_addr":"[fd00:1122:3344:101::a]:55086","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":73,"response_code":"200"}