[ Oct 15 18:33:48 Enabled. ]
[ Oct 15 18:33:48 Rereading configuration. ]
[ Oct 15 18:33:49 Rereading configuration. ]
[ Oct 15 18:33:51 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2de1de63-916d-41a7-aad9-8c2c422d0bf1/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_2de1de63-916d-41a7-aad9-8c2c422d0bf1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:51.148340839Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:51.150428873Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:51.150439803Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:51.171057808Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:51.32639298Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.741349366Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52190"}
{"msg":"region 06a6c74f-c875-4b1f-92df-ddbbcade942f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.788803437Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.800358759Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.80037902Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"uri":"/crucible/0/regions","method":"POST","req_id":"0014c140-4da2-4554-a8d6-b6f9a22d998b","remote_addr":"[fd00:1122:3344:101::a]:52190","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":25937,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_2de1de63-916d-41a7-aad9-8c2c422d0bf1/crucible/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.811270864Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_2de1de63-916d-41a7-aad9-8c2c422d0bf1/crucible/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.811304584Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"06a6c74f-c875-4b1f-92df-ddbbcade942f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.891718602Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"region":"06a6c74f-c875-4b1f-92df-ddbbcade942f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"06a6c74f-c875-4b1f-92df-ddbbcade942f\" \"--data\" \"/data/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.896320704Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"region":"06a6c74f-c875-4b1f-92df-ddbbcade942f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.052812859Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"uri":"/crucible/0/regions","method":"POST","req_id":"dc7a5ec1-6ff0-4d10-8fc3-8dac4390d4cd","remote_addr":"[fd00:1122:3344:101::a]:52190","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.31064921Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"region":"06a6c74f-c875-4b1f-92df-ddbbcade942f","component":"worker"}
{"msg":"region 06a6c74f-c875-4b1f-92df-ddbbcade942f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.310701001Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.310809652Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-06a6c74f-c875-4b1f-92df-ddbbcade942f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.313608004Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-06a6c74f-c875-4b1f-92df-ddbbcade942f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.325168226Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.351739729Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-06a6c74f-c875-4b1f-92df-ddbbcade942f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.368987946Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.369508652Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.369573532Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.369613293Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.369653453Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.377705915Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.383220598Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:40.726272721Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"uri":"/crucible/0/regions","method":"POST","req_id":"6c01518f-0a1d-41dd-812e-7adcc555b798","remote_addr":"[fd00:1122:3344:101::a]:52190","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.928679409Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"uri":"/crucible/0/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f","method":"GET","req_id":"2402782b-20ba-4b95-96be-28f897a0df32","remote_addr":"[fd00:1122:3344:101::a]:52190","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":642,"response_code":"200"}
{"msg":"path is \"/data/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.931175748Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"datafile"}
{"msg":"dataset is oxp_2de1de63-916d-41a7-aad9-8c2c422d0bf1/crucible/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.942112152Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"datafile"}
{"msg":"region 06a6c74f-c875-4b1f-92df-ddbbcade942f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.951839757Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.952001918Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.952072625Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"uri":"/crucible/0/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f","method":"DELETE","req_id":"26e2f6b6-5782-4baa-b8e8-fb6725d957c1","remote_addr":"[fd00:1122:3344:101::a]:52190","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":20989,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-06a6c74f-c875-4b1f-92df-ddbbcade942f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.953156258Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.954938318Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"uri":"/crucible/0/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f","method":"GET","req_id":"d45ea144-0985-48d8-a2dd-e5cf20df35bb","remote_addr":"[fd00:1122:3344:101::a]:52190","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.97086139Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_2de1de63-916d-41a7-aad9-8c2c422d0bf1/crucible/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.982097258Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"region":"06a6c74f-c875-4b1f-92df-ddbbcade942f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:08.011904878Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"uri":"/crucible/0/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f","method":"GET","req_id":"e03773ca-dba5-49f2-9664-f6dd9bdd068b","remote_addr":"[fd00:1122:3344:101::a]:52190","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"region 06a6c74f-c875-4b1f-92df-ddbbcade942f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:08.076466294Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:08.146834157Z","hostname":"oxz_crucible_f0098391-e25f-47ce-98db-0c870c28a2ec","pid":20264,"uri":"/crucible/0/regions/06a6c74f-c875-4b1f-92df-ddbbcade942f","method":"GET","req_id":"fa871786-e073-40c1-8af2-8df4130f728c","remote_addr":"[fd00:1122:3344:101::a]:52190","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":50,"response_code":"200"}