{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:23.816060124Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:54319"} {"msg":"region 3a262bed-bce9-4ff0-a1dc-e2645797433b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:23.816127905Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:23.816988122Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"uri":"/crucible/0/regions","method":"POST","req_id":"0bba092d-c120-45d3-a9bc-3db5ddf48fba","remote_addr":"[fd00:1122:3344:101::b]:54319","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":891,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:23.817002682Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_0bbe3c41-59de-404a-9f8e-a965d5af99fc/crucible/regions/3a262bed-bce9-4ff0-a1dc-e2645797433b","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:23.825617572Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_0bbe3c41-59de-404a-9f8e-a965d5af99fc/crucible/regions/3a262bed-bce9-4ff0-a1dc-e2645797433b","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:23.825637982Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"3a262bed-bce9-4ff0-a1dc-e2645797433b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3a262bed-bce9-4ff0-a1dc-e2645797433b\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:23.889251993Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"region":"3a262bed-bce9-4ff0-a1dc-e2645797433b","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3a262bed-bce9-4ff0-a1dc-e2645797433b\" \"--data\" \"/data/regions/3a262bed-bce9-4ff0-a1dc-e2645797433b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:23.889285124Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"region":"3a262bed-bce9-4ff0-a1dc-e2645797433b","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.001888537Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"uri":"/crucible/0/regions","method":"POST","req_id":"c09e1ca8-ef29-4ef9-8102-8fed204c589a","remote_addr":"[fd00:1122:3344:101::b]:54319","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":54,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.002977168Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"region":"3a262bed-bce9-4ff0-a1dc-e2645797433b","component":"worker"} {"msg":"region 3a262bed-bce9-4ff0-a1dc-e2645797433b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.002991189Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.003109781Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-3a262bed-bce9-4ff0-a1dc-e2645797433b","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.004434137Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3a262bed-bce9-4ff0-a1dc-e2645797433b","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.018532164Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.050531243Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3a262bed-bce9-4ff0-a1dc-e2645797433b","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.057245966Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3a262bed-bce9-4ff0-a1dc-e2645797433b","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.057582592Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.057628123Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.057669854Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.057707335Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.065692441Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.081292609Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:24.733708314Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"uri":"/crucible/0/regions","method":"POST","req_id":"a8054c9e-11f3-4e5e-8311-c457559ceb6f","remote_addr":"[fd00:1122:3344:101::b]:54319","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":58,"response_code":200} {"msg":"region 965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.679374438Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.679560132Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"uri":"/crucible/0/regions","method":"POST","req_id":"b0e0f68b-4792-4747-bc04-a93bbbe4f512","remote_addr":"[fd00:1122:3344:101::b]:54319","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":234,"response_code":200} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.679577392Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_0bbe3c41-59de-404a-9f8e-a965d5af99fc/crucible/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.686015539Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_0bbe3c41-59de-404a-9f8e-a965d5af99fc/crucible/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.686030209Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.733308538Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"region":"965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3\" \"--data\" \"/data/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.733341539Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"region":"965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.847600325Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"region":"965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","component":"worker"} {"msg":"region 965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.847630296Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.847753368Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.852663995Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.852784197Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"uri":"/crucible/0/regions","method":"POST","req_id":"0791b250-86b5-4759-bf1e-69b6e0b543bb","remote_addr":"[fd00:1122:3344:101::b]:54319","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":50,"response_code":200} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.868549437Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.881973151Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.88852588Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.888732954Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.888790345Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.888832806Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.888866316Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.897227121Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:30.90478798Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:56.910226558Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"uri":"/crucible/0/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","method":"GET","req_id":"325240c8-634f-4880-8f2e-fa8604c4b091","remote_addr":"[fd00:1122:3344:101::b]:54319","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":52,"response_code":200} {"msg":"path is \"/data/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:56.913134876Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"datafile"} {"msg":"dataset is oxp_0bbe3c41-59de-404a-9f8e-a965d5af99fc/crucible/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:56.921765755Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"datafile"} {"msg":"region 965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:56.930223982Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:56.930387505Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:56.930396105Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"uri":"/crucible/0/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","method":"DELETE","req_id":"d8c6dccc-c731-4fa7-9755-06ac1e6286f6","remote_addr":"[fd00:1122:3344:101::b]:54319","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17288,"response_code":204} {"msg":"disabling downstairs instance: downstairs-965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:56.932013337Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:56.93471289Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"uri":"/crucible/0/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","method":"GET","req_id":"b3d9fbda-9eb1-4cfd-87ea-d26f420dfdea","remote_addr":"[fd00:1122:3344:101::b]:54319","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":32,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:56.946452491Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_0bbe3c41-59de-404a-9f8e-a965d5af99fc/crucible/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:56.954400817Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"region":"965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","component":"worker"} {"msg":"region 965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:57.022305422Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-13T08:10:57.065146334Z","hostname":"oxz_crucible_1325f0d6-847d-4cd2-8177-090fc5272e04","pid":24146,"uri":"/crucible/0/regions/965bfdeb-9a29-4b38-9ad2-f95e1ab5d5e3","method":"GET","req_id":"ac769d94-c00d-45d3-9a98-6d15b59adb24","remote_addr":"[fd00:1122:3344:101::b]:54319","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":46,"response_code":200}