[ Aug 15 05:33:51 Enabled. ]
[ Aug 15 05:33:51 Rereading configuration. ]
[ Aug 15 05:33:56 Rereading configuration. ]
[ Aug 15 05:34:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:03.333778986Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:03.334728636Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:03.334737246Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:03.352189786Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:03.502214863Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.289533292Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44953"}
{"msg":"region a4477763-a3ec-4db3-823f-21679402d66d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.294231092Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.294694381Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions","method":"POST","req_id":"5ca575bd-f50c-4af1-abdb-872e639099d7","remote_addr":"[fd00:1122:3344:101::a]:44953","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":933,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.295263473Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.306793238Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.306832528Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a4477763-a3ec-4db3-823f-21679402d66d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a4477763-a3ec-4db3-823f-21679402d66d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.375238647Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"region":"a4477763-a3ec-4db3-823f-21679402d66d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a4477763-a3ec-4db3-823f-21679402d66d\" \"--data\" \"/data/regions/a4477763-a3ec-4db3-823f-21679402d66d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.375330239Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"region":"a4477763-a3ec-4db3-823f-21679402d66d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.521460654Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"region":"a4477763-a3ec-4db3-823f-21679402d66d","component":"worker"}
{"msg":"region a4477763-a3ec-4db3-823f-21679402d66d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.521518335Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.521610297Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.522029866Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.526216465Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.538288411Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.541777875Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.542482959Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.544821379Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.54487538Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.544916731Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.551989021Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.555412263Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:34:55.649574888Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions","method":"POST","req_id":"efdd6a1d-0e28-4dba-96a7-2cdafde60bc3","remote_addr":"[fd00:1122:3344:101::a]:44953","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.710938906Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52402"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.711033678Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions/a4477763-a3ec-4db3-823f-21679402d66d","method":"GET","req_id":"24381ecd-7e06-46cd-891f-42f6bdfcec3f","remote_addr":"[fd00:1122:3344:101::a]:52402","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"path is \"/data/regions/a4477763-a3ec-4db3-823f-21679402d66d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.716756269Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"dataset is oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.726157749Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"snapshot is oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d@2f033b20-288f-4dfe-bc32-e77aeb582e4b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.734975546Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d\", \"2f033b20-288f-4dfe-bc32-e77aeb582e4b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.735005926Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.746667923Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions/a4477763-a3ec-4db3-823f-21679402d66d/snapshots/2f033b20-288f-4dfe-bc32-e77aeb582e4b","method":"GET","req_id":"3ceb08f7-3c91-44ae-b4c6-d862eaf3db24","remote_addr":"[fd00:1122:3344:101::a]:52402","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":32871,"response_code":"200"}
{"msg":"path is \"/data/regions/a4477763-a3ec-4db3-823f-21679402d66d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.749683687Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"dataset is oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.758388831Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"snapshot is oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d@2f033b20-288f-4dfe-bc32-e77aeb582e4b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.766434082Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d\", \"2f033b20-288f-4dfe-bc32-e77aeb582e4b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.766468162Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot a4477763-a3ec-4db3-823f-21679402d66d-2f033b20-288f-4dfe-bc32-e77aeb582e4b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.775970764Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"applying SMF actions for region a4477763-a3ec-4db3-823f-21679402d66d running snapshot 2f033b20-288f-4dfe-bc32-e77aeb582e4b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.776600277Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.776624968Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions/a4477763-a3ec-4db3-823f-21679402d66d/snapshots/2f033b20-288f-4dfe-bc32-e77aeb582e4b/run","method":"POST","req_id":"eab75154-5339-42b6-b92c-9a2dbda72ebf","remote_addr":"[fd00:1122:3344:101::a]:52402","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26974,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.782259327Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.782347359Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.782431231Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.782438151Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.782482342Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.782533093Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.782630485Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.782696936Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.782745627Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.782796378Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.788353606Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-a4477763-a3ec-4db3-823f-21679402d66d-2f033b20-288f-4dfe-bc32-e77aeb582e4b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.788819316Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-a4477763-a3ec-4db3-823f-21679402d66d-2f033b20-288f-4dfe-bc32-e77aeb582e4b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.791795139Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-a4477763-a3ec-4db3-823f-21679402d66d-2f033b20-288f-4dfe-bc32-e77aeb582e4b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.801415393Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a4477763-a3ec-4db3-823f-21679402d66d/.zfs/snapshot/2f033b20-288f-4dfe-bc32-e77aeb582e4b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.801689088Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.80174568Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.80178943Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.801831211Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.801885383Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.805116421Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.80933515Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"region a4477763-a3ec-4db3-823f-21679402d66d running snapshot 2f033b20-288f-4dfe-bc32-e77aeb582e4b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.809351681Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"path is \"/data/regions/a4477763-a3ec-4db3-823f-21679402d66d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.91178751Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"dataset is oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.921400044Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"snapshot is oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d@2f033b20-288f-4dfe-bc32-e77aeb582e4b","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.929472825Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/a4477763-a3ec-4db3-823f-21679402d66d\", \"2f033b20-288f-4dfe-bc32-e77aeb582e4b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.929502406Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:21.937876963Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions/a4477763-a3ec-4db3-823f-21679402d66d/snapshots/2f033b20-288f-4dfe-bc32-e77aeb582e4b/run","method":"POST","req_id":"31c6c83b-2ce7-4517-87d8-e6e5ace1f448","remote_addr":"[fd00:1122:3344:101::a]:52402","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26116,"response_code":"200"}
{"msg":"region b9c6d847-d77c-40f0-8a96-b4b07b2f7663 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:24.923262043Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:24.923424786Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions","method":"POST","req_id":"18a9989e-1d49-4983-8626-9f90a63dbfe9","remote_addr":"[fd00:1122:3344:101::a]:52402","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":230,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:24.923453437Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:24.933152472Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:24.933182993Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b9c6d847-d77c-40f0-8a96-b4b07b2f7663\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:24.994564633Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"region":"b9c6d847-d77c-40f0-8a96-b4b07b2f7663","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b9c6d847-d77c-40f0-8a96-b4b07b2f7663\" \"--data\" \"/data/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:24.994627484Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"region":"b9c6d847-d77c-40f0-8a96-b4b07b2f7663","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.06620719Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"region":"b9c6d847-d77c-40f0-8a96-b4b07b2f7663","component":"worker"}
{"msg":"region b9c6d847-d77c-40f0-8a96-b4b07b2f7663 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.066265662Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.066355963Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.066947216Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.067028058Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.067112259Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.0671192Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.06716297Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.067213631Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.06759569Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.067660561Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.067714902Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.067766713Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.072467773Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b9c6d847-d77c-40f0-8a96-b4b07b2f7663","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.072927383Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b9c6d847-d77c-40f0-8a96-b4b07b2f7663","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.083991907Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.091277141Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b9c6d847-d77c-40f0-8a96-b4b07b2f7663","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.093910637Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.094234204Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.094291345Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.094336666Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.094377267Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.097728018Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.103219104Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:25.234866572Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions","method":"POST","req_id":"b8636961-faed-4853-8b26-1ab1c5ba642f","remote_addr":"[fd00:1122:3344:101::a]:52402","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.577458329Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663","method":"GET","req_id":"7cdc9897-52c2-4f1c-b338-697abedd97bd","remote_addr":"[fd00:1122:3344:101::a]:52402","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"path is \"/data/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.580190697Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"dataset is oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.591781792Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"region b9c6d847-d77c-40f0-8a96-b4b07b2f7663 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.600816694Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.601014448Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.601070189Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663","method":"DELETE","req_id":"3309f713-9da9-4100-a4a9-c3eab6306004","remote_addr":"[fd00:1122:3344:101::a]:52402","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20915,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-b9c6d847-d77c-40f0-8a96-b4b07b2f7663 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.602226484Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.603905549Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663","method":"GET","req_id":"d32e2b49-33ba-435c-853a-783e71b33482","remote_addr":"[fd00:1122:3344:101::a]:52402","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.606322191Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.606414082Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.606502174Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.606512924Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.606553125Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.606608197Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a4477763-a3ec-4db3-823f-21679402d66d","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.606981705Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.607043706Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.607096217Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.607147368Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.610622382Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.612104363Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_620567fb-982d-4cc1-a4b5-d88c7fee0672/crucible/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.622545324Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"region":"b9c6d847-d77c-40f0-8a96-b4b07b2f7663","component":"worker"}
{"msg":"region b9c6d847-d77c-40f0-8a96-b4b07b2f7663 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.670066341Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-15T05:40:48.721777436Z","hostname":"oxz_crucible_7f76d969-4b23-4248-9341-ab938cef91d1","pid":22307,"uri":"/crucible/0/regions/b9c6d847-d77c-40f0-8a96-b4b07b2f7663","method":"GET","req_id":"072bb496-47a9-456b-a1a8-733e7fcacc06","remote_addr":"[fd00:1122:3344:101::a]:52402","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":86,"response_code":"200"}