{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.598013659Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53805"} {"msg":"region a21495f9-6c96-457c-b7f1-b0be46ffb133 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.598143822Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.599077884Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"uri":"/crucible/0/regions","method":"POST","req_id":"cacdcbee-dc18-49c9-bfde-fecb857075ae","remote_addr":"[fd00:1122:3344:101::a]:53805","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":1019,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.599100295Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/a21495f9-6c96-457c-b7f1-b0be46ffb133","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.610158355Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/a21495f9-6c96-457c-b7f1-b0be46ffb133","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.610185056Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"a21495f9-6c96-457c-b7f1-b0be46ffb133\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a21495f9-6c96-457c-b7f1-b0be46ffb133\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.65583993Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"region":"a21495f9-6c96-457c-b7f1-b0be46ffb133","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a21495f9-6c96-457c-b7f1-b0be46ffb133\" \"--data\" \"/data/regions/a21495f9-6c96-457c-b7f1-b0be46ffb133\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.657784696Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"region":"a21495f9-6c96-457c-b7f1-b0be46ffb133","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.768416289Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"region":"a21495f9-6c96-457c-b7f1-b0be46ffb133","component":"worker"} {"msg":"region a21495f9-6c96-457c-b7f1-b0be46ffb133 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.76845724Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.768602453Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-a21495f9-6c96-457c-b7f1-b0be46ffb133","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.774525893Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a21495f9-6c96-457c-b7f1-b0be46ffb133","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.78546857Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.793029178Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a21495f9-6c96-457c-b7f1-b0be46ffb133","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.796378387Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a21495f9-6c96-457c-b7f1-b0be46ffb133","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.796680644Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.796734715Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.796778026Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.796808227Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.800288209Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.804648072Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:03:58.885944514Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"uri":"/crucible/0/regions","method":"POST","req_id":"15034567-dc8e-4809-9d71-da3f671aa03a","remote_addr":"[fd00:1122:3344:101::a]:53805","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":80,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.366463209Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"uri":"/crucible/0/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5","method":"GET","req_id":"4c33506c-62f4-4ed5-8cbc-1e4681a0e7f1","remote_addr":"[fd00:1122:3344:101::a]:53805","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":58,"response_code":"200"} {"msg":"path is \"/data/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.370073334Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"datafile"} {"msg":"dataset is oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.379598058Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"datafile"} {"msg":"snapshot is oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5@ae859332-3ded-4d8d-b1d6-4b3257ef154b","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.388260982Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5\", \"ae859332-3ded-4d8d-b1d6-4b3257ef154b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.388295283Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.398548854Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"uri":"/crucible/0/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5/snapshots/ae859332-3ded-4d8d-b1d6-4b3257ef154b","method":"GET","req_id":"f8d1e116-daf0-4e84-afe6-27699848fcff","remote_addr":"[fd00:1122:3344:101::a]:53805","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":28474,"response_code":"200"} {"msg":"path is \"/data/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.401830691Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"datafile"} {"msg":"dataset is oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.409737807Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"datafile"} {"msg":"snapshot is oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5@ae859332-3ded-4d8d-b1d6-4b3257ef154b","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.418424162Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5\", \"ae859332-3ded-4d8d-b1d6-4b3257ef154b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.418447372Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 0c3b33b5-f12d-480f-8e11-16241c50ffe5-ae859332-3ded-4d8d-b1d6-4b3257ef154b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.426968853Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"datafile"} {"msg":"applying SMF actions for region 0c3b33b5-f12d-480f-8e11-16241c50ffe5 running snapshot ae859332-3ded-4d8d-b1d6-4b3257ef154b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.427175117Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.427204728Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"uri":"/crucible/0/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5/snapshots/ae859332-3ded-4d8d-b1d6-4b3257ef154b/run","method":"POST","req_id":"d7a3f010-432a-4299-b906-fd8ddabdbc24","remote_addr":"[fd00:1122:3344:101::a]:53805","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25390,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-0c3b33b5-f12d-480f-8e11-16241c50ffe5-ae859332-3ded-4d8d-b1d6-4b3257ef154b","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.429970873Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-0c3b33b5-f12d-480f-8e11-16241c50ffe5-ae859332-3ded-4d8d-b1d6-4b3257ef154b","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.433045146Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-0c3b33b5-f12d-480f-8e11-16241c50ffe5-ae859332-3ded-4d8d-b1d6-4b3257ef154b","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.443583304Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5/.zfs/snapshot/ae859332-3ded-4d8d-b1d6-4b3257ef154b","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.44385918Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.443910351Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.443952382Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.443993673Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.444055035Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.447952176Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.452169716Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"worker"} {"msg":"region 0c3b33b5-f12d-480f-8e11-16241c50ffe5 running snapshot ae859332-3ded-4d8d-b1d6-4b3257ef154b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.452179356Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"datafile"} {"msg":"path is \"/data/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.601198812Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"datafile"} {"msg":"dataset is oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.610362868Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"datafile"} {"msg":"snapshot is oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5@ae859332-3ded-4d8d-b1d6-4b3257ef154b","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.618600732Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_3be8e8db-d2f3-4573-9b35-b81f32ee0e80/crucible/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5\", \"ae859332-3ded-4d8d-b1d6-4b3257ef154b\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.618624422Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:04:00.626659971Z","hostname":"oxz_crucible_17fb43eb-3013-4b13-91ee-97860ef1b9f1","pid":22825,"uri":"/crucible/0/regions/0c3b33b5-f12d-480f-8e11-16241c50ffe5/snapshots/ae859332-3ded-4d8d-b1d6-4b3257ef154b/run","method":"POST","req_id":"96bf6bb4-31e8-437c-a6ae-a97172f0feba","remote_addr":"[fd00:1122:3344:101::a]:53805","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25471,"response_code":"200"}