{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.341940786Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56410"}
{"msg":"region 461a1d43-5247-4dd1-90bb-1d7dc8be372e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.399969508Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.40783215Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.412845773Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"uri":"/crucible/0/regions","method":"POST","req_id":"ca391fda-751e-44fd-89f4-840b8667d867","remote_addr":"[fd00:1122:3344:101::a]:56410","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21443,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_8cb1c0ad-1492-43b1-ad64-5587c984b90a/crucible/regions/461a1d43-5247-4dd1-90bb-1d7dc8be372e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.423159685Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8cb1c0ad-1492-43b1-ad64-5587c984b90a/crucible/regions/461a1d43-5247-4dd1-90bb-1d7dc8be372e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.423176335Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"461a1d43-5247-4dd1-90bb-1d7dc8be372e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/461a1d43-5247-4dd1-90bb-1d7dc8be372e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.494983741Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"region":"461a1d43-5247-4dd1-90bb-1d7dc8be372e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"461a1d43-5247-4dd1-90bb-1d7dc8be372e\" \"--data\" \"/data/regions/461a1d43-5247-4dd1-90bb-1d7dc8be372e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.49789907Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"region":"461a1d43-5247-4dd1-90bb-1d7dc8be372e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.774094656Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"uri":"/crucible/0/regions","method":"POST","req_id":"3e0c842e-41cd-4c39-9f44-71646b8df401","remote_addr":"[fd00:1122:3344:101::a]:56410","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.84927173Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"region":"461a1d43-5247-4dd1-90bb-1d7dc8be372e","component":"worker"}
{"msg":"region 461a1d43-5247-4dd1-90bb-1d7dc8be372e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.849307901Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.849484185Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-461a1d43-5247-4dd1-90bb-1d7dc8be372e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.850061417Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-461a1d43-5247-4dd1-90bb-1d7dc8be372e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.861087153Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.882974863Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-461a1d43-5247-4dd1-90bb-1d7dc8be372e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.88719314Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/461a1d43-5247-4dd1-90bb-1d7dc8be372e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.888022557Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.888090148Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.888136169Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.8881778Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.893869267Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:22.901248409Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:23.320697177Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"uri":"/crucible/0/regions","method":"POST","req_id":"2dd9d8ea-ad0a-4946-9e50-ae662362db87","remote_addr":"[fd00:1122:3344:101::a]:56410","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"region bf29d107-6b39-4f0c-82c1-0310124143b6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.634165709Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.634346162Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"uri":"/crucible/0/regions","method":"POST","req_id":"ef76b810-f108-4a9f-9616-453ba723a664","remote_addr":"[fd00:1122:3344:101::a]:56410","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":230,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.634361413Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_8cb1c0ad-1492-43b1-ad64-5587c984b90a/crucible/regions/bf29d107-6b39-4f0c-82c1-0310124143b6","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.644072392Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8cb1c0ad-1492-43b1-ad64-5587c984b90a/crucible/regions/bf29d107-6b39-4f0c-82c1-0310124143b6","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.644100363Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"bf29d107-6b39-4f0c-82c1-0310124143b6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bf29d107-6b39-4f0c-82c1-0310124143b6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.71697771Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"region":"bf29d107-6b39-4f0c-82c1-0310124143b6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bf29d107-6b39-4f0c-82c1-0310124143b6\" \"--data\" \"/data/regions/bf29d107-6b39-4f0c-82c1-0310124143b6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.717042872Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"region":"bf29d107-6b39-4f0c-82c1-0310124143b6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.799585498Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"uri":"/crucible/0/regions","method":"POST","req_id":"06939947-c464-43fe-b0e2-29b8ef713de8","remote_addr":"[fd00:1122:3344:101::a]:56410","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":547,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.835938515Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"region":"bf29d107-6b39-4f0c-82c1-0310124143b6","component":"worker"}
{"msg":"region bf29d107-6b39-4f0c-82c1-0310124143b6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.835982126Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.836103828Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-bf29d107-6b39-4f0c-82c1-0310124143b6","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.840437277Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bf29d107-6b39-4f0c-82c1-0310124143b6","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.846380769Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.858262803Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bf29d107-6b39-4f0c-82c1-0310124143b6","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.86489315Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bf29d107-6b39-4f0c-82c1-0310124143b6","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.865179256Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.865232967Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.865276978Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.865316458Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.870453944Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:28.878182523Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:29.126032435Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"uri":"/crucible/0/regions","method":"POST","req_id":"81790b32-c735-4344-9e67-d0649714c51c","remote_addr":"[fd00:1122:3344:101::a]:56410","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":90,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.243255762Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"uri":"/crucible/0/regions/bf29d107-6b39-4f0c-82c1-0310124143b6","method":"GET","req_id":"2b5c12d3-519b-4753-8248-ce2f85db1356","remote_addr":"[fd00:1122:3344:101::a]:56410","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"path is \"/data/regions/bf29d107-6b39-4f0c-82c1-0310124143b6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.247236974Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"datafile"}
{"msg":"dataset is oxp_8cb1c0ad-1492-43b1-ad64-5587c984b90a/crucible/regions/bf29d107-6b39-4f0c-82c1-0310124143b6","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.258992346Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"datafile"}
{"msg":"region bf29d107-6b39-4f0c-82c1-0310124143b6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.26990608Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.270094874Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.270143705Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"uri":"/crucible/0/regions/bf29d107-6b39-4f0c-82c1-0310124143b6","method":"DELETE","req_id":"f15d5d10-042e-4018-992e-6d6a285fa8bd","remote_addr":"[fd00:1122:3344:101::a]:56410","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22930,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-bf29d107-6b39-4f0c-82c1-0310124143b6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.271190096Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.273534265Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"uri":"/crucible/0/regions/bf29d107-6b39-4f0c-82c1-0310124143b6","method":"GET","req_id":"fa295ddd-8e2a-4c87-b123-dfd984d0a54e","remote_addr":"[fd00:1122:3344:101::a]:56410","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.276776661Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8cb1c0ad-1492-43b1-ad64-5587c984b90a/crucible/regions/bf29d107-6b39-4f0c-82c1-0310124143b6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.288539943Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"region":"bf29d107-6b39-4f0c-82c1-0310124143b6","component":"worker"}
{"msg":"region bf29d107-6b39-4f0c-82c1-0310124143b6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.3404473Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-03T23:24:55.422779331Z","hostname":"oxz_crucible_562bf5e5-6372-49fb-9d89-d8f7b0d2f198","pid":21338,"uri":"/crucible/0/regions/bf29d107-6b39-4f0c-82c1-0310124143b6","method":"GET","req_id":"013c1817-6c09-42a8-9032-e2bbc56f215f","remote_addr":"[fd00:1122:3344:101::a]:56410","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":672,"response_code":"200"}