{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.216266231Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:64764"} {"msg":"region e72141c9-aa64-4a87-8624-da41ec585450 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.274577898Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.28247303Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.28249021Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions","method":"POST","req_id":"b64ed047-bd83-44e2-82b5-46c6a37cfe09","remote_addr":"[fd00:1122:3344:101::b]:64764","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25929,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_d988f988-2649-4d71-a2a0-0f194156e959/crucible/regions/e72141c9-aa64-4a87-8624-da41ec585450","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.29541805Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_d988f988-2649-4d71-a2a0-0f194156e959/crucible/regions/e72141c9-aa64-4a87-8624-da41ec585450","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.295438441Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e72141c9-aa64-4a87-8624-da41ec585450\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e72141c9-aa64-4a87-8624-da41ec585450\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.366317101Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"region":"e72141c9-aa64-4a87-8624-da41ec585450","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e72141c9-aa64-4a87-8624-da41ec585450\" \"--data\" \"/data/regions/e72141c9-aa64-4a87-8624-da41ec585450\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.369427441Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"region":"e72141c9-aa64-4a87-8624-da41ec585450","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.475385458Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions","method":"POST","req_id":"441a0950-015f-450f-a810-3af9baf51577","remote_addr":"[fd00:1122:3344:101::b]:64764","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":99,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.795509914Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"region":"e72141c9-aa64-4a87-8624-da41ec585450","component":"worker"} {"msg":"region e72141c9-aa64-4a87-8624-da41ec585450 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.795561615Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.795649817Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e72141c9-aa64-4a87-8624-da41ec585450","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.796024724Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.803940327Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions","method":"POST","req_id":"a3ed6c46-3b6f-421e-987c-f46505c6e788","remote_addr":"[fd00:1122:3344:101::b]:64764","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":55,"response_code":200} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e72141c9-aa64-4a87-8624-da41ec585450","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.809408492Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.834402105Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e72141c9-aa64-4a87-8624-da41ec585450","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.838677208Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e72141c9-aa64-4a87-8624-da41ec585450","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.839517584Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.839596636Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.839634447Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.839668907Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.844319887Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:48.849985347Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"region e46fe8af-2d00-4d6c-9908-b288f3ca4b1c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.452952375Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.453117849Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.453130499Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions","method":"POST","req_id":"2042271f-74cd-4007-9b0f-9cc40a7b2229","remote_addr":"[fd00:1122:3344:101::b]:64764","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":227,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_d988f988-2649-4d71-a2a0-0f194156e959/crucible/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.459885349Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_d988f988-2649-4d71-a2a0-0f194156e959/crucible/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.45990962Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e46fe8af-2d00-4d6c-9908-b288f3ca4b1c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.588379342Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"region":"e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e46fe8af-2d00-4d6c-9908-b288f3ca4b1c\" \"--data\" \"/data/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.588461874Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"region":"e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.665553263Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions","method":"POST","req_id":"a065778c-bf45-4fe6-902a-eb5b24a54cb0","remote_addr":"[fd00:1122:3344:101::b]:64764","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":86,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.862231754Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"region":"e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","component":"worker"} {"msg":"region e46fe8af-2d00-4d6c-9908-b288f3ca4b1c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.862301576Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.862469249Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.862946838Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.870248199Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.89668835Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.914843931Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.91532676Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.915386171Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.915444072Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.915493693Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.9334611Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.940430825Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions","method":"POST","req_id":"ff0b7a44-4483-4e27-88f9-e277ebd7ba5d","remote_addr":"[fd00:1122:3344:101::b]:64764","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":66,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:56.948220366Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:57.181374121Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:44115"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:57.181527174Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","method":"GET","req_id":"2d761507-4530-4693-9f9d-720012e1bfd8","remote_addr":"[fd00:1122:3344:101::c]:44115","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":71,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:57.25015846Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:45096"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:24:57.250255022Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","method":"GET","req_id":"8af40def-03dd-422a-80ee-8cce1ca3cdc4","remote_addr":"[fd00:1122:3344:101::b]:45096","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":38,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.850766625Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:55353"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.85106615Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","method":"GET","req_id":"31fab277-5552-4ed7-a2e2-7a40bf3dd97a","remote_addr":"[fd00:1122:3344:101::b]:55353","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":52,"response_code":200} {"msg":"path is \"/data/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.855265581Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"datafile"} {"msg":"dataset is oxp_d988f988-2649-4d71-a2a0-0f194156e959/crucible/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.866104131Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"datafile"} {"msg":"region e46fe8af-2d00-4d6c-9908-b288f3ca4b1c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.873735138Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.873891191Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.873901361Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","method":"DELETE","req_id":"36a69b62-fc52-4670-9aaf-0cfb5e708db4","remote_addr":"[fd00:1122:3344:101::b]:55353","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18668,"response_code":204} {"msg":"disabling downstairs instance: downstairs-e46fe8af-2d00-4d6c-9908-b288f3ca4b1c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.874818739Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.876988171Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","method":"GET","req_id":"1166032f-454f-4b1a-ad7c-3c27fb667c19","remote_addr":"[fd00:1122:3344:101::b]:55353","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":31,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.891699685Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_d988f988-2649-4d71-a2a0-0f194156e959/crucible/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.900179759Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"region":"e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","component":"worker"} {"msg":"region e46fe8af-2d00-4d6c-9908-b288f3ca4b1c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.982588752Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-03T20:25:31.986031898Z","hostname":"oxz_crucible_d408999e-da87-4527-baea-870fb836d56f","pid":23542,"uri":"/crucible/0/regions/e46fe8af-2d00-4d6c-9908-b288f3ca4b1c","method":"GET","req_id":"9f5bfc71-7874-4f83-999c-4dfcec595c73","remote_addr":"[fd00:1122:3344:101::b]:55353","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":48,"response_code":200}