[ Aug 3 04:36:44 Enabled. ] [ Aug 3 04:36:44 Rereading configuration. ] [ Aug 3 04:36:50 Rereading configuration. ] [ Aug 3 04:36:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:36:54.131227648Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496} {"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:36:54.13243266Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:36:54.132446061Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:36:54.153715569Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:36:54.299663459Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.408826981Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54141"} {"msg":"region e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.436915734Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.444326673Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.448184235Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions","method":"POST","req_id":"1bd3059a-c5b3-4b67-8c18-ae36b81a0b8c","remote_addr":"[fd00:1122:3344:101::a]:54141","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":19205,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.455733096Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.455764606Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.511444235Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"region":"e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8\" \"--data\" \"/data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.513997912Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"region":"e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.601811337Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions","method":"POST","req_id":"646642de-d90f-4b79-b3e8-bae1cfc6b37c","remote_addr":"[fd00:1122:3344:101::a]:54141","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":67,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.82434687Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"region":"e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","component":"worker"} {"msg":"region e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.824396071Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.824569573Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.825102609Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.830360515Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.847047265Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.857169153Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.860286907Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.862792544Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.862863005Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.862942096Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.867610466Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:09.87171069Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:38:10.24644822Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions","method":"POST","req_id":"ba0ce726-b76b-4e5d-8957-9bf9be223c09","remote_addr":"[fd00:1122:3344:101::a]:54141","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.035441547Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55744"} {"msg":"region 534b0b61-ebb3-467a-a154-9158f9afad9a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.035554288Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.036215976Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.036232336Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions","method":"POST","req_id":"3880ea7d-ed04-4deb-be75-2243d4919439","remote_addr":"[fd00:1122:3344:101::a]:55744","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":740,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/534b0b61-ebb3-467a-a154-9158f9afad9a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.047477189Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/534b0b61-ebb3-467a-a154-9158f9afad9a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.047571518Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"534b0b61-ebb3-467a-a154-9158f9afad9a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/534b0b61-ebb3-467a-a154-9158f9afad9a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.114408756Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"region":"534b0b61-ebb3-467a-a154-9158f9afad9a","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"534b0b61-ebb3-467a-a154-9158f9afad9a\" \"--data\" \"/data/regions/534b0b61-ebb3-467a-a154-9158f9afad9a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.114466677Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"region":"534b0b61-ebb3-467a-a154-9158f9afad9a","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.194857352Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"region":"534b0b61-ebb3-467a-a154-9158f9afad9a","component":"worker"} {"msg":"region 534b0b61-ebb3-467a-a154-9158f9afad9a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.194901672Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.194985183Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-534b0b61-ebb3-467a-a154-9158f9afad9a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.195458388Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-534b0b61-ebb3-467a-a154-9158f9afad9a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.200503102Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.215916508Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-534b0b61-ebb3-467a-a154-9158f9afad9a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.222524769Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/534b0b61-ebb3-467a-a154-9158f9afad9a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.222833152Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.222890413Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.222933354Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.222972694Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.228387782Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.234021343Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.234107264Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.234192005Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.234200175Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.234240915Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.234290886Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.234365747Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.234425077Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.234478128Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.234531808Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.237648552Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions","method":"POST","req_id":"11b9f1a0-a829-48ac-991f-7205d4fec05b","remote_addr":"[fd00:1122:3344:101::a]:55744","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":82,"response_code":"200"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.24582307Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.246406076Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.739021684Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45007"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.739127045Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","method":"GET","req_id":"1615c0cd-76f8-468f-8adc-6af8a334f193","remote_addr":"[fd00:1122:3344:101::a]:45007","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":60,"response_code":"200"} {"msg":"path is \"/data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.742817955Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"dataset is oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.752491489Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"snapshot is oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8@b65cb9c8-ff1b-4ef9-a79a-705765fb831b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.761694448Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8\", \"b65cb9c8-ff1b-4ef9-a79a-705765fb831b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.761732899Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.773170832Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8/snapshots/b65cb9c8-ff1b-4ef9-a79a-705765fb831b","method":"GET","req_id":"55818e76-af2b-469f-9a53-296d32f03279","remote_addr":"[fd00:1122:3344:101::a]:45007","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":33860,"response_code":"200"} {"msg":"path is \"/data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.773320483Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"dataset is oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.782113668Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"snapshot is oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8@b65cb9c8-ff1b-4ef9-a79a-705765fb831b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.790584399Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8\", \"b65cb9c8-ff1b-4ef9-a79a-705765fb831b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.790615619Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8-b65cb9c8-ff1b-4ef9-a79a-705765fb831b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.798867808Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"applying SMF actions for region e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8 running snapshot b65cb9c8-ff1b-4ef9-a79a-705765fb831b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.7990733Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.79908772Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8/snapshots/b65cb9c8-ff1b-4ef9-a79a-705765fb831b/run","method":"POST","req_id":"68b513a2-8b5f-48fe-a098-6e03ee12dfce","remote_addr":"[fd00:1122:3344:101::a]:45007","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25788,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.800836419Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.80091943Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.801004601Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.801014631Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.801055511Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.801102832Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.801426046Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.801485076Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.801534407Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.801583427Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.806073796Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8-b65cb9c8-ff1b-4ef9-a79a-705765fb831b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.806592761Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8-b65cb9c8-ff1b-4ef9-a79a-705765fb831b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.816428377Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8-b65cb9c8-ff1b-4ef9-a79a-705765fb831b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.828426587Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8/.zfs/snapshot/b65cb9c8-ff1b-4ef9-a79a-705765fb831b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.82877602Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.82884467Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.828887461Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.828934081Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.828990542Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.834884985Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.843269685Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"region e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8 running snapshot b65cb9c8-ff1b-4ef9-a79a-705765fb831b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:37.843283616Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"region 5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.219006001Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.219142682Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions","method":"POST","req_id":"4bf3d85c-4a50-410f-92d9-7aee01ac697c","remote_addr":"[fd00:1122:3344:101::a]:55744","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":205,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.219160732Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.22824925Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.228287081Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.28400194Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"region":"5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79\" \"--data\" \"/data/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.284059661Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"region":"5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.357639912Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"region":"5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","component":"worker"} {"msg":"region 5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.357688882Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.357773593Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.358999947Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.362546185Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.37050339Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.373780455Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.374076419Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.374140799Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.37419155Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.374243951Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.378965831Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.384018776Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.384123077Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.384220688Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.384227948Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.384269128Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.384315919Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.389015669Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.38907167Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.389120771Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.389165961Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.39276331Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.394795581Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:43:41.575772938Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions","method":"POST","req_id":"766a04c7-ab31-48fd-8324-d37dd69c3118","remote_addr":"[fd00:1122:3344:101::a]:55744","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.158067936Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","method":"GET","req_id":"61b66b23-0a96-46b3-a40d-60e3dd65b48a","remote_addr":"[fd00:1122:3344:101::a]:55744","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":63,"response_code":"200"} {"msg":"path is \"/data/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.160816355Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"dataset is oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.172545021Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"region 5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.181960233Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.182183395Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.182245106Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","method":"DELETE","req_id":"9d3a672c-e390-4a4d-9b2e-133d0abab4d3","remote_addr":"[fd00:1122:3344:101::a]:55744","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":21453,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.183755942Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.18539959Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","method":"GET","req_id":"8f1cf0b7-e480-486e-9b6a-d9e57e5bbda7","remote_addr":"[fd00:1122:3344:101::a]:55744","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":28,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.18932348Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.189415918Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.189503791Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.189510382Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.189553545Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.189605567Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e5e45ede-9f3e-4578-9bc0-2ed90f1d0ea8","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.19010756Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.190166171Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.190215862Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.190264332Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.195129124Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.19662934Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_869fe861-6756-4391-9071-05e75ac72c2e/crucible/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.20680699Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"region":"5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","component":"worker"} {"msg":"region 5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.258042381Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:44:06.265680243Z","hostname":"oxz_crucible_2975dcbc-e1db-4b8d-a51b-98f501c26c48","pid":21496,"uri":"/crucible/0/regions/5b7a0ca7-c1c3-4df3-93a3-01fa5de5ef79","method":"GET","req_id":"bc987019-d22e-4e03-ba0a-155062c8f6e4","remote_addr":"[fd00:1122:3344:101::a]:55744","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":67,"response_code":"200"}