{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:41.903445969Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49595"} {"msg":"region 6e778c96-0117-4e61-8bae-f95e66265ead state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:41.955479781Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:41.960555383Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"uri":"/crucible/0/regions","method":"POST","req_id":"50d55588-0ff8-47f1-a9dc-8fc17412ed0c","remote_addr":"[fd00:1122:3344:101::a]:49595","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":23512,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:41.960577554Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_981f05bd-27a7-444e-9407-bfa49f8c5c88/crucible/regions/6e778c96-0117-4e61-8bae-f95e66265ead","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:41.972214945Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_981f05bd-27a7-444e-9407-bfa49f8c5c88/crucible/regions/6e778c96-0117-4e61-8bae-f95e66265ead","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:41.972239706Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"6e778c96-0117-4e61-8bae-f95e66265ead\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6e778c96-0117-4e61-8bae-f95e66265ead\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.033187733Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"region":"6e778c96-0117-4e61-8bae-f95e66265ead","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6e778c96-0117-4e61-8bae-f95e66265ead\" \"--data\" \"/data/regions/6e778c96-0117-4e61-8bae-f95e66265ead\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.035353302Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"region":"6e778c96-0117-4e61-8bae-f95e66265ead","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.273223149Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"uri":"/crucible/0/regions","method":"POST","req_id":"649706e6-4562-42f7-887e-d0bb2a007f7c","remote_addr":"[fd00:1122:3344:101::a]:49595","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":110,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.382879621Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"region":"6e778c96-0117-4e61-8bae-f95e66265ead","component":"worker"} {"msg":"region 6e778c96-0117-4e61-8bae-f95e66265ead state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.382926362Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.383046796Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-6e778c96-0117-4e61-8bae-f95e66265ead","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.383528662Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6e778c96-0117-4e61-8bae-f95e66265ead","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.393978186Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.41166758Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6e778c96-0117-4e61-8bae-f95e66265ead","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.414418268Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6e778c96-0117-4e61-8bae-f95e66265ead","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.414995457Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.415056569Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.41509784Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.415140111Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.418792548Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.435666727Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:42.882473078Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"uri":"/crucible/0/regions","method":"POST","req_id":"197f9015-0488-478c-8160-db770fb2a690","remote_addr":"[fd00:1122:3344:101::a]:49595","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":49,"response_code":"200"} {"msg":"region 30da2e62-e538-4f30-9d14-565d8557e3c4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.520602837Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.520788394Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"uri":"/crucible/0/regions","method":"POST","req_id":"88fc80c0-0b59-4386-9231-3f2cdbc3243a","remote_addr":"[fd00:1122:3344:101::a]:49595","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":238,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.520805444Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_981f05bd-27a7-444e-9407-bfa49f8c5c88/crucible/regions/30da2e62-e538-4f30-9d14-565d8557e3c4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.530494444Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_981f05bd-27a7-444e-9407-bfa49f8c5c88/crucible/regions/30da2e62-e538-4f30-9d14-565d8557e3c4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.530527254Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"30da2e62-e538-4f30-9d14-565d8557e3c4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/30da2e62-e538-4f30-9d14-565d8557e3c4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.589307792Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"region":"30da2e62-e538-4f30-9d14-565d8557e3c4","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"30da2e62-e538-4f30-9d14-565d8557e3c4\" \"--data\" \"/data/regions/30da2e62-e538-4f30-9d14-565d8557e3c4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.589373024Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"region":"30da2e62-e538-4f30-9d14-565d8557e3c4","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.662626344Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"region":"30da2e62-e538-4f30-9d14-565d8557e3c4","component":"worker"} {"msg":"region 30da2e62-e538-4f30-9d14-565d8557e3c4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.662676715Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.662760328Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-30da2e62-e538-4f30-9d14-565d8557e3c4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.663157781Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-30da2e62-e538-4f30-9d14-565d8557e3c4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.665696182Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.671793086Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-30da2e62-e538-4f30-9d14-565d8557e3c4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.674438391Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/30da2e62-e538-4f30-9d14-565d8557e3c4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.674684899Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.67473694Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.674780172Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.674818783Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.678135509Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.683460369Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:04:48.84346148Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"uri":"/crucible/0/regions","method":"POST","req_id":"9e07fdae-fdfc-4c39-94e2-a508709f29a3","remote_addr":"[fd00:1122:3344:101::a]:49595","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":59,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.197577559Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"uri":"/crucible/0/regions/30da2e62-e538-4f30-9d14-565d8557e3c4","method":"GET","req_id":"64abbe22-4d29-4110-b984-238b249e4d47","remote_addr":"[fd00:1122:3344:101::a]:49595","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":2564,"response_code":"200"} {"msg":"path is \"/data/regions/30da2e62-e538-4f30-9d14-565d8557e3c4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.200926635Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"datafile"} {"msg":"dataset is oxp_981f05bd-27a7-444e-9407-bfa49f8c5c88/crucible/regions/30da2e62-e538-4f30-9d14-565d8557e3c4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.212295789Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"datafile"} {"msg":"region 30da2e62-e538-4f30-9d14-565d8557e3c4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.22297061Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.223161766Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.223214018Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"uri":"/crucible/0/regions/30da2e62-e538-4f30-9d14-565d8557e3c4","method":"DELETE","req_id":"6e2fc386-3b8d-4ed9-a8b7-9ac60f196716","remote_addr":"[fd00:1122:3344:101::a]:49595","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22927,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-30da2e62-e538-4f30-9d14-565d8557e3c4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.224278481Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.22642796Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"uri":"/crucible/0/regions/30da2e62-e538-4f30-9d14-565d8557e3c4","method":"GET","req_id":"3a2e77b6-7428-4b08-8216-b24502befc9a","remote_addr":"[fd00:1122:3344:101::a]:49595","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":28,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.230754198Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_981f05bd-27a7-444e-9407-bfa49f8c5c88/crucible/regions/30da2e62-e538-4f30-9d14-565d8557e3c4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.242588686Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"region":"30da2e62-e538-4f30-9d14-565d8557e3c4","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.289816655Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"uri":"/crucible/0/regions/30da2e62-e538-4f30-9d14-565d8557e3c4","method":"GET","req_id":"21be7680-b8ec-4a06-adc2-909517641ffa","remote_addr":"[fd00:1122:3344:101::a]:49595","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":82,"response_code":"200"} {"msg":"region 30da2e62-e538-4f30-9d14-565d8557e3c4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.304596717Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T06:05:13.415656544Z","hostname":"oxz_crucible_8ded696c-35ea-4517-af23-80b4fa3ca95f","pid":23063,"uri":"/crucible/0/regions/30da2e62-e538-4f30-9d14-565d8557e3c4","method":"GET","req_id":"3043b602-a68b-48c0-9204-f578a0193559","remote_addr":"[fd00:1122:3344:101::a]:49595","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":31,"response_code":"200"}