{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.054324436Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:36340"} {"msg":"region 583a594a-74dc-4beb-9b4e-e8cb68fa5be9 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.076153983Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.081354165Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.081409796Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"uri":"/crucible/0/regions","method":"POST","req_id":"48053b82-fe71-4561-9837-58fb50524ace","remote_addr":"[fd00:1122:3344:101::a]:36340","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":13820,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.095723308Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.095751469Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"583a594a-74dc-4beb-9b4e-e8cb68fa5be9\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.15543401Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"region":"583a594a-74dc-4beb-9b4e-e8cb68fa5be9","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"583a594a-74dc-4beb-9b4e-e8cb68fa5be9\" \"--data\" \"/data/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.157337442Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"region":"583a594a-74dc-4beb-9b4e-e8cb68fa5be9","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.299704045Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"region":"583a594a-74dc-4beb-9b4e-e8cb68fa5be9","component":"worker"} {"msg":"region 583a594a-74dc-4beb-9b4e-e8cb68fa5be9 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.299761736Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.299839539Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.300241249Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.305095232Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.319970729Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.322820277Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.328143262Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.330348273Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.330405334Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.330446655Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.33429483Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.338072154Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:30:40.448219255Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"uri":"/crucible/0/regions","method":"POST","req_id":"17cc9955-7189-4fcb-a2ce-96d85ae63103","remote_addr":"[fd00:1122:3344:101::a]:36340","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":638,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.691812094Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:48619"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.691896126Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"uri":"/crucible/0/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9","method":"GET","req_id":"a6300291-31c9-4eb6-ae7c-34e4257ddcc6","remote_addr":"[fd00:1122:3344:101::a]:48619","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":58,"response_code":"200"} {"msg":"path is \"/data/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.699593927Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"datafile"} {"msg":"dataset is oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.708367446Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"datafile"} {"msg":"snapshot is oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9@e10522a9-60d2-4910-96af-df41fd7d982a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.716699884Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9\", \"e10522a9-60d2-4910-96af-df41fd7d982a\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.716734615Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.728741423Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"uri":"/crucible/0/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9/snapshots/e10522a9-60d2-4910-96af-df41fd7d982a","method":"GET","req_id":"e8962f4c-bea8-43be-8c16-36e45b8ef0c7","remote_addr":"[fd00:1122:3344:101::a]:48619","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":31616,"response_code":"200"} {"msg":"path is \"/data/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.7319185Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"datafile"} {"msg":"dataset is oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.740585627Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"datafile"} {"msg":"snapshot is oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9@e10522a9-60d2-4910-96af-df41fd7d982a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.74907755Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9\", \"e10522a9-60d2-4910-96af-df41fd7d982a\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.74910834Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 583a594a-74dc-4beb-9b4e-e8cb68fa5be9-e10522a9-60d2-4910-96af-df41fd7d982a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.75750605Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"datafile"} {"msg":"applying SMF actions for region 583a594a-74dc-4beb-9b4e-e8cb68fa5be9 running snapshot e10522a9-60d2-4910-96af-df41fd7d982a (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.758358533Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.758390424Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"uri":"/crucible/0/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9/snapshots/e10522a9-60d2-4910-96af-df41fd7d982a/run","method":"POST","req_id":"6e0a5267-3465-4a49-8b91-dd5a19295d46","remote_addr":"[fd00:1122:3344:101::a]:48619","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":26505,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.764701417Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.764786029Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.764867581Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.764875391Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.764917452Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.764976694Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.765076547Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.765138539Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.76518683Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.765236091Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.771964435Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-583a594a-74dc-4beb-9b4e-e8cb68fa5be9-e10522a9-60d2-4910-96af-df41fd7d982a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.772414328Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-583a594a-74dc-4beb-9b4e-e8cb68fa5be9-e10522a9-60d2-4910-96af-df41fd7d982a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.778520584Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-583a594a-74dc-4beb-9b4e-e8cb68fa5be9-e10522a9-60d2-4910-96af-df41fd7d982a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.794013358Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9/.zfs/snapshot/e10522a9-60d2-4910-96af-df41fd7d982a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.794259725Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.794324347Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.794373248Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.794422269Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.794481781Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.798698906Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.805155873Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"worker"} {"msg":"region 583a594a-74dc-4beb-9b4e-e8cb68fa5be9 running snapshot e10522a9-60d2-4910-96af-df41fd7d982a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:53.805181693Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"datafile"} {"msg":"path is \"/data/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:54.060148244Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"datafile"} {"msg":"dataset is oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:54.069481869Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"datafile"} {"msg":"snapshot is oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9@e10522a9-60d2-4910-96af-df41fd7d982a","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:54.078032473Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_74ef6b74-2e66-4a72-b4f9-068f5d8802cb/crucible/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9\", \"e10522a9-60d2-4910-96af-df41fd7d982a\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:54.078069024Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-16T21:35:54.08634478Z","hostname":"oxz_crucible_dcff56d7-ff94-4732-abab-da244575f545","pid":21174,"uri":"/crucible/0/regions/583a594a-74dc-4beb-9b4e-e8cb68fa5be9/snapshots/e10522a9-60d2-4910-96af-df41fd7d982a/run","method":"POST","req_id":"a311ccb3-f887-48af-8d47-e8a0604ac355","remote_addr":"[fd00:1122:3344:101::a]:48619","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":26231,"response_code":"200"}