{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:43.590660279Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:45286"} {"msg":"region a143ecc5-d562-4917-904e-89ae25c0bce1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:43.633400259Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:43.638648605Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"uri":"/crucible/0/regions","method":"POST","req_id":"0310172f-c912-4869-99c8-f6f1aa48dcb9","remote_addr":"[fd00:1122:3344:101::c]:45286","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24048,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:43.641167805Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_cecd40b2-dad7-4ec4-bda4-8ca88537bfa9/crucible/regions/a143ecc5-d562-4917-904e-89ae25c0bce1","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:43.655048312Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_cecd40b2-dad7-4ec4-bda4-8ca88537bfa9/crucible/regions/a143ecc5-d562-4917-904e-89ae25c0bce1","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:43.655076143Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"a143ecc5-d562-4917-904e-89ae25c0bce1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a143ecc5-d562-4917-904e-89ae25c0bce1\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:43.717391509Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"region":"a143ecc5-d562-4917-904e-89ae25c0bce1","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a143ecc5-d562-4917-904e-89ae25c0bce1\" \"--data\" \"/data/regions/a143ecc5-d562-4917-904e-89ae25c0bce1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:43.71744561Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"region":"a143ecc5-d562-4917-904e-89ae25c0bce1","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:43.808258339Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"uri":"/crucible/0/regions","method":"POST","req_id":"86b8f14e-00bf-4def-9982-2544c1b51c0c","remote_addr":"[fd00:1122:3344:101::c]:45286","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.037377391Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"region":"a143ecc5-d562-4917-904e-89ae25c0bce1","component":"worker"} {"msg":"region a143ecc5-d562-4917-904e-89ae25c0bce1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.037423342Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.037540596Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-a143ecc5-d562-4917-904e-89ae25c0bce1","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.037983158Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a143ecc5-d562-4917-904e-89ae25c0bce1","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.049844488Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.071907273Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a143ecc5-d562-4917-904e-89ae25c0bce1","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.075314108Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a143ecc5-d562-4917-904e-89ae25c0bce1","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.076053738Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.07610956Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.076148911Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.076184162Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.080487852Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.098982047Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:44.521363961Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"uri":"/crucible/0/regions","method":"POST","req_id":"7c1c07b9-f345-46f9-a9a6-97da40f5912b","remote_addr":"[fd00:1122:3344:101::c]:45286","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":46,"response_code":"200"} {"msg":"region bf049819-f9cb-4ab9-bca0-7797e84c4afb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:49.992985946Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:49.993154813Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"uri":"/crucible/0/regions","method":"POST","req_id":"5ee3afaa-576a-46bf-a82b-22ce3c934feb","remote_addr":"[fd00:1122:3344:101::c]:45286","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":217,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:49.993168472Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_cecd40b2-dad7-4ec4-bda4-8ca88537bfa9/crucible/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.002936878Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_cecd40b2-dad7-4ec4-bda4-8ca88537bfa9/crucible/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.002964889Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"bf049819-f9cb-4ab9-bca0-7797e84c4afb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.067424625Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"region":"bf049819-f9cb-4ab9-bca0-7797e84c4afb","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bf049819-f9cb-4ab9-bca0-7797e84c4afb\" \"--data\" \"/data/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.067486946Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"region":"bf049819-f9cb-4ab9-bca0-7797e84c4afb","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.148508123Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"region":"bf049819-f9cb-4ab9-bca0-7797e84c4afb","component":"worker"} {"msg":"region bf049819-f9cb-4ab9-bca0-7797e84c4afb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.148555244Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.148645717Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-bf049819-f9cb-4ab9-bca0-7797e84c4afb","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.149993174Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bf049819-f9cb-4ab9-bca0-7797e84c4afb","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.153163463Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.160706403Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bf049819-f9cb-4ab9-bca0-7797e84c4afb","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.16383327Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.164144668Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.16419581Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.164236511Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.164270732Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.168404107Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.173299763Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:38:50.349959484Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"uri":"/crucible/0/regions","method":"POST","req_id":"0b7522ae-94c0-4f06-bb17-6550368e14f4","remote_addr":"[fd00:1122:3344:101::c]:45286","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":58,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.120120744Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"uri":"/crucible/0/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb","method":"GET","req_id":"f18529e6-1365-4618-a7af-84b7049de84c","remote_addr":"[fd00:1122:3344:101::c]:45286","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":91,"response_code":"200"} {"msg":"path is \"/data/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.122373437Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"datafile"} {"msg":"dataset is oxp_cecd40b2-dad7-4ec4-bda4-8ca88537bfa9/crucible/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.138706292Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"datafile"} {"msg":"region bf049819-f9cb-4ab9-bca0-7797e84c4afb state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.148626818Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.148971598Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-bf049819-f9cb-4ab9-bca0-7797e84c4afb (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.150052248Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.151359574Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"uri":"/crucible/0/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb","method":"DELETE","req_id":"f2c4d757-f18e-4e30-91bf-0addff5a8a73","remote_addr":"[fd00:1122:3344:101::c]:45286","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":29000,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.154485211Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"uri":"/crucible/0/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb","method":"GET","req_id":"18cf0743-fc29-4215-b453-c9ac65e6558c","remote_addr":"[fd00:1122:3344:101::c]:45286","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.168737908Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_cecd40b2-dad7-4ec4-bda4-8ca88537bfa9/crucible/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.181061311Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"region":"bf049819-f9cb-4ab9-bca0-7797e84c4afb","component":"worker"} {"msg":"region bf049819-f9cb-4ab9-bca0-7797e84c4afb state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.235144518Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-01T10:39:15.308392678Z","hostname":"oxz_crucible_e79eaa07-7b4e-4ce7-aef6-d50212d4e28e","pid":23500,"uri":"/crucible/0/regions/bf049819-f9cb-4ab9-bca0-7797e84c4afb","method":"GET","req_id":"b5d68b1c-c118-45ea-843f-690e96d9670c","remote_addr":"[fd00:1122:3344:101::c]:45286","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":54,"response_code":"200"}