{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.202823984Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:41622"} {"msg":"region 4ffe31c8-bcd0-41d4-994d-ff87c3526feb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.202911535Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.203864492Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.203898703Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"uri":"/crucible/0/regions","method":"POST","req_id":"067dc734-c585-4dce-a984-4c49f343b6b5","remote_addr":"[fd00:1122:3344:101::b]:41622","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":979,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_801a87fa-21d0-48f9-8b62-72f340f5e88b/crucible/regions/4ffe31c8-bcd0-41d4-994d-ff87c3526feb","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.214323389Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_801a87fa-21d0-48f9-8b62-72f340f5e88b/crucible/regions/4ffe31c8-bcd0-41d4-994d-ff87c3526feb","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.21435444Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"4ffe31c8-bcd0-41d4-994d-ff87c3526feb\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4ffe31c8-bcd0-41d4-994d-ff87c3526feb\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.256951751Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"region":"4ffe31c8-bcd0-41d4-994d-ff87c3526feb","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4ffe31c8-bcd0-41d4-994d-ff87c3526feb\" \"--data\" \"/data/regions/4ffe31c8-bcd0-41d4-994d-ff87c3526feb\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.256991741Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"region":"4ffe31c8-bcd0-41d4-994d-ff87c3526feb","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.309763264Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"region":"4ffe31c8-bcd0-41d4-994d-ff87c3526feb","component":"worker"} {"msg":"region 4ffe31c8-bcd0-41d4-994d-ff87c3526feb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.309796575Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.309907957Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-4ffe31c8-bcd0-41d4-994d-ff87c3526feb","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.310804923Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4ffe31c8-bcd0-41d4-994d-ff87c3526feb","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.320023727Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.334259892Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4ffe31c8-bcd0-41d4-994d-ff87c3526feb","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.335605506Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4ffe31c8-bcd0-41d4-994d-ff87c3526feb","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.33581608Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.33586421Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.335900251Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.335936222Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.337808935Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.340673836Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:43.541987243Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"uri":"/crucible/0/regions","method":"POST","req_id":"4202f41a-4ad3-4bc2-8821-c946d5571bb5","remote_addr":"[fd00:1122:3344:101::b]:41622","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":62,"response_code":200} {"msg":"region dd38b8a6-929b-4d38-b609-a7d3f14196ec state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.317555481Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.317775795Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.317789595Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"uri":"/crucible/0/regions","method":"POST","req_id":"90f09d92-d2d6-48c9-8636-13a89b4d4039","remote_addr":"[fd00:1122:3344:101::b]:41622","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":287,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_801a87fa-21d0-48f9-8b62-72f340f5e88b/crucible/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.324999394Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_801a87fa-21d0-48f9-8b62-72f340f5e88b/crucible/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.325032534Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"dd38b8a6-929b-4d38-b609-a7d3f14196ec\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.367281689Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"region":"dd38b8a6-929b-4d38-b609-a7d3f14196ec","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"dd38b8a6-929b-4d38-b609-a7d3f14196ec\" \"--data\" \"/data/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.36731958Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"region":"dd38b8a6-929b-4d38-b609-a7d3f14196ec","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.42613887Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"region":"dd38b8a6-929b-4d38-b609-a7d3f14196ec","component":"worker"} {"msg":"region dd38b8a6-929b-4d38-b609-a7d3f14196ec state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.426173441Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.426281843Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-dd38b8a6-929b-4d38-b609-a7d3f14196ec","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.431659759Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dd38b8a6-929b-4d38-b609-a7d3f14196ec","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.43900079Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.442853809Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dd38b8a6-929b-4d38-b609-a7d3f14196ec","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.444348716Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.44459266Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.444636151Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.444685391Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.444725272Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.446478734Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.449045049Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:07:49.60527705Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"uri":"/crucible/0/regions","method":"POST","req_id":"65225445-f4b7-48be-bbab-736e9b400033","remote_addr":"[fd00:1122:3344:101::b]:41622","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.386067527Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"uri":"/crucible/0/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec","method":"GET","req_id":"e1981731-0d50-471d-8feb-a032ad13cdd8","remote_addr":"[fd00:1122:3344:101::b]:41622","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":56,"response_code":200} {"msg":"path is \"/data/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.38956742Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"datafile"} {"msg":"dataset is oxp_801a87fa-21d0-48f9-8b62-72f340f5e88b/crucible/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.40135029Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"datafile"} {"msg":"region dd38b8a6-929b-4d38-b609-a7d3f14196ec state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.408413156Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.408587989Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.40861835Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"uri":"/crucible/0/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec","method":"DELETE","req_id":"40227730-a1ff-4f69-bb12-37a701d08ada","remote_addr":"[fd00:1122:3344:101::b]:41622","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":19072,"response_code":204} {"msg":"disabling downstairs instance: downstairs-dd38b8a6-929b-4d38-b609-a7d3f14196ec (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.409867122Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.412388287Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"uri":"/crucible/0/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec","method":"GET","req_id":"8623b3f2-c2ff-485c-9874-d74d4a65b932","remote_addr":"[fd00:1122:3344:101::b]:41622","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":54,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.413929315Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_801a87fa-21d0-48f9-8b62-72f340f5e88b/crucible/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec\"","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.422395986Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"region":"dd38b8a6-929b-4d38-b609-a7d3f14196ec","component":"worker"} {"msg":"region dd38b8a6-929b-4d38-b609-a7d3f14196ec state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.464583209Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-05-22T22:08:14.48307233Z","hostname":"oxz_crucible_fcaebca5-f39b-49ac-a14d-f796fb6d52ae","pid":21772,"uri":"/crucible/0/regions/dd38b8a6-929b-4d38-b609-a7d3f14196ec","method":"GET","req_id":"b3703c62-fbcb-4abc-9ea9-875b99b19abd","remote_addr":"[fd00:1122:3344:101::b]:41622","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":49,"response_code":200}