{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:52.707941935Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62549"} {"msg":"region 7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:52.708039256Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:52.7086423Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"uri":"/crucible/0/regions","method":"POST","req_id":"76e42c70-45e1-47f7-ad91-981a6e11d9a0","remote_addr":"[fd00:1122:3344:101::a]:62549","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":645,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:52.7086581Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_3554a27f-cc97-4d5f-b65f-742b39508465/crucible/regions/7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:52.720635807Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3554a27f-cc97-4d5f-b65f-742b39508465/crucible/regions/7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:52.720665378Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:52.794314806Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"region":"7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1\" \"--data\" \"/data/regions/7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:52.794386476Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"region":"7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.080672234Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"uri":"/crucible/0/regions","method":"POST","req_id":"9d1cd668-d328-4763-a039-792f65fff2fd","remote_addr":"[fd00:1122:3344:101::a]:62549","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":116,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.102390375Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"region":"7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1","component":"worker"} {"msg":"region 7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.102428656Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.102536676Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.104003396Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.111862757Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.133070635Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.136102584Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7b17a6b1-9918-44ce-8d3a-4a4ea36aaac1","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.136398706Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.136447746Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.136488317Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.136527577Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.139967579Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.144401158Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:53.415681109Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"uri":"/crucible/0/regions","method":"POST","req_id":"b0974cf7-1ddd-4d37-a086-f33066388c62","remote_addr":"[fd00:1122:3344:101::a]:62549","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":84,"response_code":"200"} {"msg":"region 60ea46eb-9904-48b2-a224-79c04c3d8d35 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.199361326Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.199550418Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.199567368Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"uri":"/crucible/0/regions","method":"POST","req_id":"6e2b0502-3790-4be5-b693-2e51ee2ed5d1","remote_addr":"[fd00:1122:3344:101::a]:62549","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":268,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_3554a27f-cc97-4d5f-b65f-742b39508465/crucible/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.208608066Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3554a27f-cc97-4d5f-b65f-742b39508465/crucible/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.208635087Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"60ea46eb-9904-48b2-a224-79c04c3d8d35\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.267928102Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"region":"60ea46eb-9904-48b2-a224-79c04c3d8d35","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"60ea46eb-9904-48b2-a224-79c04c3d8d35\" \"--data\" \"/data/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.267976462Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"region":"60ea46eb-9904-48b2-a224-79c04c3d8d35","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.338778111Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"region":"60ea46eb-9904-48b2-a224-79c04c3d8d35","component":"worker"} {"msg":"region 60ea46eb-9904-48b2-a224-79c04c3d8d35 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.338827702Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.338978373Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-60ea46eb-9904-48b2-a224-79c04c3d8d35","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.34009292Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-60ea46eb-9904-48b2-a224-79c04c3d8d35","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.347673089Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.354584144Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-60ea46eb-9904-48b2-a224-79c04c3d8d35","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.357447343Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.357750494Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.357800765Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.357845955Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.357885126Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.361429649Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.366950134Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:53:58.433601667Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"uri":"/crucible/0/regions","method":"POST","req_id":"f30d742c-0b56-49e7-9d9e-25b57845e396","remote_addr":"[fd00:1122:3344:101::a]:62549","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":85,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.117363261Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"uri":"/crucible/0/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35","method":"GET","req_id":"7754b6b5-2b48-4b77-a3ca-885c5b15c7cf","remote_addr":"[fd00:1122:3344:101::a]:62549","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":77,"response_code":"200"} {"msg":"path is \"/data/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.121136636Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"datafile"} {"msg":"dataset is oxp_3554a27f-cc97-4d5f-b65f-742b39508465/crucible/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.134471662Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"datafile"} {"msg":"region 60ea46eb-9904-48b2-a224-79c04c3d8d35 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.143863453Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.144088775Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.144157725Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"uri":"/crucible/0/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35","method":"DELETE","req_id":"00fe9c00-b1c4-4451-94b9-038f2fa174d7","remote_addr":"[fd00:1122:3344:101::a]:62549","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":23053,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-60ea46eb-9904-48b2-a224-79c04c3d8d35 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.145748086Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.150398456Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"uri":"/crucible/0/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35","method":"GET","req_id":"d9291685-c915-4ee6-ac80-3dc116c75eab","remote_addr":"[fd00:1122:3344:101::a]:62549","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":854,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.15101684Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_3554a27f-cc97-4d5f-b65f-742b39508465/crucible/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.162735526Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"region":"60ea46eb-9904-48b2-a224-79c04c3d8d35","component":"worker"} {"msg":"region 60ea46eb-9904-48b2-a224-79c04c3d8d35 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.226663631Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:54:25.262174361Z","hostname":"oxz_crucible_7db095ef-0c07-408e-a294-6f51716fa8a4","pid":23838,"uri":"/crucible/0/regions/60ea46eb-9904-48b2-a224-79c04c3d8d35","method":"GET","req_id":"b26eccd5-ad0e-4472-b933-3db62611f4c1","remote_addr":"[fd00:1122:3344:101::a]:62549","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":86,"response_code":"200"}