{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.110983242Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:49469"} {"msg":"region a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.153539546Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.158542259Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.163714148Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"uri":"/crucible/0/regions","method":"POST","req_id":"6b57b6fc-82eb-44d3-baa9-12811ee7dfd1","remote_addr":"[fd00:1122:3344:101::b]:49469","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":16998,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_5d7ad0a3-e47e-4369-b599-d02dbd3a8829/crucible/regions/a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.172058508Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5d7ad0a3-e47e-4369-b599-d02dbd3a8829/crucible/regions/a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.172069999Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.232769304Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"region":"a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7\" \"--data\" \"/data/regions/a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.235345618Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"region":"a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.491481495Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"uri":"/crucible/0/regions","method":"POST","req_id":"9bb11c16-c605-4dc5-ac00-529a1222863c","remote_addr":"[fd00:1122:3344:101::b]:49469","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":77,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.556383072Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"region":"a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7","component":"worker"} {"msg":"region a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.556431593Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.556589408Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.557119433Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.573812163Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.589871655Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.594401545Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a54eb4c4-e1ee-44f1-9d3c-090f686ab5f7","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.595430145Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.595526958Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.59559989Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.595661491Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.601766507Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.607928334Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:49.780758805Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"uri":"/crucible/0/regions","method":"POST","req_id":"84c1fc9b-690d-4762-ab41-3f7f484d6fc3","remote_addr":"[fd00:1122:3344:101::b]:49469","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":91,"response_code":"200"} {"msg":"region f7b4fbd9-58b0-4a08-9933-61b23bea30f5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.376311859Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.376470704Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"uri":"/crucible/0/regions","method":"POST","req_id":"ab6c5891-d029-49ed-b728-8ae5d1d6d452","remote_addr":"[fd00:1122:3344:101::b]:49469","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":207,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.376488224Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_5d7ad0a3-e47e-4369-b599-d02dbd3a8829/crucible/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.386842162Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5d7ad0a3-e47e-4369-b599-d02dbd3a8829/crucible/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.386874133Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f7b4fbd9-58b0-4a08-9933-61b23bea30f5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.464230958Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"region":"f7b4fbd9-58b0-4a08-9933-61b23bea30f5","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f7b4fbd9-58b0-4a08-9933-61b23bea30f5\" \"--data\" \"/data/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.466137152Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"region":"f7b4fbd9-58b0-4a08-9933-61b23bea30f5","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.557534141Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"region":"f7b4fbd9-58b0-4a08-9933-61b23bea30f5","component":"worker"} {"msg":"region f7b4fbd9-58b0-4a08-9933-61b23bea30f5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.557574362Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.557672565Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f7b4fbd9-58b0-4a08-9933-61b23bea30f5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.561423663Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.56270885Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"uri":"/crucible/0/regions","method":"POST","req_id":"7dc1bdbb-f1c3-4f2f-887f-0ed98aa70eb6","remote_addr":"[fd00:1122:3344:101::b]:49469","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":70,"response_code":"200"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f7b4fbd9-58b0-4a08-9933-61b23bea30f5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.564913773Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.57556577Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f7b4fbd9-58b0-4a08-9933-61b23bea30f5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.5793871Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.579593056Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.579638497Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.579679398Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.579718429Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.585603209Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:07:54.591065046Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.777597712Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"uri":"/crucible/0/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5","method":"GET","req_id":"89edfd04-3725-42e8-9f53-d17ef3224d77","remote_addr":"[fd00:1122:3344:101::b]:49469","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":83,"response_code":"200"} {"msg":"path is \"/data/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.77961237Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"datafile"} {"msg":"dataset is oxp_5d7ad0a3-e47e-4369-b599-d02dbd3a8829/crucible/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.790355448Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"datafile"} {"msg":"region f7b4fbd9-58b0-4a08-9933-61b23bea30f5 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.800048167Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.800230432Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.800292564Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"uri":"/crucible/0/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5","method":"DELETE","req_id":"ee6117fe-4592-4621-a946-8dd22ac14aaa","remote_addr":"[fd00:1122:3344:101::b]:49469","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":20703,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-f7b4fbd9-58b0-4a08-9933-61b23bea30f5 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.803465396Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.804447804Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"uri":"/crucible/0/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5","method":"GET","req_id":"c6c77e86-beca-4877-88e1-0d506df54978","remote_addr":"[fd00:1122:3344:101::b]:49469","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":29,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.810402855Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_5d7ad0a3-e47e-4369-b599-d02dbd3a8829/crucible/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.820858606Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"region":"f7b4fbd9-58b0-4a08-9933-61b23bea30f5","component":"worker"} {"msg":"region f7b4fbd9-58b0-4a08-9933-61b23bea30f5 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.895672928Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T22:08:18.950792283Z","hostname":"oxz_crucible_6e9711ba-8ebd-4b53-ad66-96b22d084d7b","pid":22921,"uri":"/crucible/0/regions/f7b4fbd9-58b0-4a08-9933-61b23bea30f5","method":"GET","req_id":"6441f111-58cb-428e-b052-d97432adeb49","remote_addr":"[fd00:1122:3344:101::b]:49469","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":71,"response_code":"200"}