[ Sep 11 22:50:26 Enabled. ] [ Sep 11 22:50:26 Rereading configuration. ] [ Sep 11 22:50:29 Rereading configuration. ] [ Sep 11 22:50:35 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d700c1e0-2586-4b27-b46c-83bfe687db08/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_d700c1e0-2586-4b27-b46c-83bfe687db08/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:50:35.843019708Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526} {"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:50:35.845559773Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:50:35.845573063Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:50:35.859319626Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:50:36.023270026Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.037155022Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58607"} {"msg":"region 32b4ac8e-189f-486b-9c52-9119fe0f7889 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.079586563Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.085806033Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions","method":"POST","req_id":"88c4ddd5-6385-4a3e-be6c-abd19bc73010","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":29277,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.092634814Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_d700c1e0-2586-4b27-b46c-83bfe687db08/crucible/regions/32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.10370417Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_d700c1e0-2586-4b27-b46c-83bfe687db08/crucible/regions/32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.10373295Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"32b4ac8e-189f-486b-9c52-9119fe0f7889\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/32b4ac8e-189f-486b-9c52-9119fe0f7889\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.171759513Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"region":"32b4ac8e-189f-486b-9c52-9119fe0f7889","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"32b4ac8e-189f-486b-9c52-9119fe0f7889\" \"--data\" \"/data/regions/32b4ac8e-189f-486b-9c52-9119fe0f7889\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.176796712Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"region":"32b4ac8e-189f-486b-9c52-9119fe0f7889","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.266141672Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions","method":"POST","req_id":"ca11ebeb-a8b7-4ffb-a7ba-0ca34aa960f0","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":83,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.588043295Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"region":"32b4ac8e-189f-486b-9c52-9119fe0f7889","component":"worker"} {"msg":"region 32b4ac8e-189f-486b-9c52-9119fe0f7889 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.588096816Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.588205508Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.588707327Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.595898324Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.615323638Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.620785694Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.635408533Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.640303049Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.640363621Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.640402711Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.647022328Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.658198216Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:16.766612983Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions","method":"POST","req_id":"5829d02a-a714-4e59-9f0c-ae6ef3d0c356","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":66,"response_code":"200"} {"msg":"region 87e484bf-9a39-47f1-b990-4b67c4cbb142 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.527726673Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.527914606Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions","method":"POST","req_id":"d20890f2-9eae-4746-aff0-709a4d1adfae","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":934,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.527931127Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_d700c1e0-2586-4b27-b46c-83bfe687db08/crucible/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.53829898Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_d700c1e0-2586-4b27-b46c-83bfe687db08/crucible/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.538337471Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"87e484bf-9a39-47f1-b990-4b67c4cbb142\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.63498378Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"region":"87e484bf-9a39-47f1-b990-4b67c4cbb142","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"87e484bf-9a39-47f1-b990-4b67c4cbb142\" \"--data\" \"/data/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.636954255Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"region":"87e484bf-9a39-47f1-b990-4b67c4cbb142","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.719943052Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions","method":"POST","req_id":"09392b04-ec4c-48d7-9a38-bafa0de31dc0","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":446,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.739245774Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"region":"87e484bf-9a39-47f1-b990-4b67c4cbb142","component":"worker"} {"msg":"region 87e484bf-9a39-47f1-b990-4b67c4cbb142 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.739270274Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.739389336Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.740620848Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.74073311Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.740839002Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.740847452Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.740905403Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.740996195Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.741113537Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.743518709Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.7435713Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.743623631Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.751687804Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-87e484bf-9a39-47f1-b990-4b67c4cbb142","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.752113671Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-87e484bf-9a39-47f1-b990-4b67c4cbb142","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.757813782Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.770488886Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-87e484bf-9a39-47f1-b990-4b67c4cbb142","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.776798458Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.777107443Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.777165744Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.777213645Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.777257456Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.782097672Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:21.78881391Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:22.075980179Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions","method":"POST","req_id":"f7743eb0-3821-4b4b-b6a5-6aef844f53ab","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":65,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.454912715Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142","method":"GET","req_id":"76c23c27-daa8-4628-8dc1-efbb49f58490","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":92,"response_code":"200"} {"msg":"path is \"/data/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.458539959Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"datafile"} {"msg":"dataset is oxp_d700c1e0-2586-4b27-b46c-83bfe687db08/crucible/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.477766429Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"datafile"} {"msg":"region 87e484bf-9a39-47f1-b990-4b67c4cbb142 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.486454663Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.486655446Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.486718737Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142","method":"DELETE","req_id":"02c2c564-ef14-4574-a68e-c772d97ec37f","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":28208,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-87e484bf-9a39-47f1-b990-4b67c4cbb142 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.487717355Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.490198289Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142","method":"GET","req_id":"13d6297f-e7b5-4f02-9f0a-231eba25ea27","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":32,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.500789476Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.500882928Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.500969759Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.50098724Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.50101792Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.501076111Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/32b4ac8e-189f-486b-9c52-9119fe0f7889","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.501524179Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.50159325Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.501641951Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.501689112Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.526133074Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.526718275Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_d700c1e0-2586-4b27-b46c-83bfe687db08/crucible/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.535401668Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"region":"87e484bf-9a39-47f1-b990-4b67c4cbb142","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.593398754Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142","method":"GET","req_id":"39ccc2e6-3c97-4028-ab31-2c0091d58c96","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":931,"response_code":"200"} {"msg":"region 87e484bf-9a39-47f1-b990-4b67c4cbb142 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.610457916Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-11T22:57:49.700215913Z","hostname":"oxz_crucible_64ce5226-a6ee-4705-8a1c-d6a6d9056277","pid":22526,"uri":"/crucible/0/regions/87e484bf-9a39-47f1-b990-4b67c4cbb142","method":"GET","req_id":"48388a64-ebe3-4bdf-877f-9a2e8729f939","remote_addr":"[fd00:1122:3344:101::a]:58607","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":29,"response_code":"200"}