[ Nov  9 02:46:15 Enabled. ]
[ Nov  9 02:46:15 Rereading configuration. ]
[ Nov  9 02:46:16 Rereading configuration. ]
[ Nov  9 02:46:17 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:18.017995007Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:18.021927308Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:18.021948627Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:18.045495325Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:46:18.132177667Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:12.978753762Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:54285"}
{"msg":"region 5c2df4be-e07c-4023-9bcb-a538bdc5e80f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:12.999814147Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.003625316Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.008465536Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions","method":"POST","req_id":"852c418b-ec08-43cb-8217-262e0bd38877","remote_addr":"[fd00:1122:3344:101::c]:54285","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":15073,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.015579403Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.015612244Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5c2df4be-e07c-4023-9bcb-a538bdc5e80f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.07542104Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"region":"5c2df4be-e07c-4023-9bcb-a538bdc5e80f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5c2df4be-e07c-4023-9bcb-a538bdc5e80f\" \"--data\" \"/data/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.079656238Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"region":"5c2df4be-e07c-4023-9bcb-a538bdc5e80f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.148549281Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions","method":"POST","req_id":"7df50ba0-6a9c-4b38-ab45-152e5ba36933","remote_addr":"[fd00:1122:3344:101::c]:54285","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.302901191Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"region":"5c2df4be-e07c-4023-9bcb-a538bdc5e80f","component":"worker"}
{"msg":"region 5c2df4be-e07c-4023-9bcb-a538bdc5e80f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.302954622Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.303054934Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5c2df4be-e07c-4023-9bcb-a538bdc5e80f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.303455113Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5c2df4be-e07c-4023-9bcb-a538bdc5e80f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.307625969Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.324645261Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5c2df4be-e07c-4023-9bcb-a538bdc5e80f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.328388458Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.32896189Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.331246257Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.331311178Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.331363989Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.340318394Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.345185335Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.425066965Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions","method":"POST","req_id":"0184070b-1534-4075-bf6b-8f88be72b8c6","remote_addr":"[fd00:1122:3344:101::c]:54285","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.728638969Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f","method":"GET","req_id":"393272dc-0967-47b1-8b09-395a8fcf629f","remote_addr":"[fd00:1122:3344:101::c]:54285","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"path is \"/data/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.73207489Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"dataset is oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.743379094Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"region 5c2df4be-e07c-4023-9bcb-a538bdc5e80f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.753298049Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.753503603Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-5c2df4be-e07c-4023-9bcb-a538bdc5e80f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.754868861Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.755669768Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f","method":"DELETE","req_id":"b1940416-a462-4688-ad41-86710c4e78ef","remote_addr":"[fd00:1122:3344:101::c]:54285","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23618,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.759380325Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f","method":"GET","req_id":"110f6251-763d-445d-aca1-e04fea136b81","remote_addr":"[fd00:1122:3344:101::c]:54285","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":34,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.759987507Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.770189218Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"region":"5c2df4be-e07c-4023-9bcb-a538bdc5e80f","component":"worker"}
{"msg":"region 5c2df4be-e07c-4023-9bcb-a538bdc5e80f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.828678477Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:13.901295007Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions/5c2df4be-e07c-4023-9bcb-a538bdc5e80f","method":"GET","req_id":"f15b8e5f-9a61-45fd-a667-0680c54a419e","remote_addr":"[fd00:1122:3344:101::c]:54285","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.127452766Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44943"}
{"msg":"region 6eb94268-1054-4982-832b-831b223e834b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.127624519Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.127796973Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions","method":"POST","req_id":"f438bbfa-20e7-4cca-85ae-98aa9b6c7050","remote_addr":"[fd00:1122:3344:101::a]:44943","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":214,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.127813073Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.137140956Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.137180247Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6eb94268-1054-4982-832b-831b223e834b\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6eb94268-1054-4982-832b-831b223e834b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.191346646Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"region":"6eb94268-1054-4982-832b-831b223e834b","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6eb94268-1054-4982-832b-831b223e834b\" \"--data\" \"/data/regions/6eb94268-1054-4982-832b-831b223e834b\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.19346485Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"region":"6eb94268-1054-4982-832b-831b223e834b","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.262105439Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"region":"6eb94268-1054-4982-832b-831b223e834b","component":"worker"}
{"msg":"region 6eb94268-1054-4982-832b-831b223e834b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.262144679Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.262244662Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6eb94268-1054-4982-832b-831b223e834b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.263688691Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6eb94268-1054-4982-832b-831b223e834b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.266911408Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.274244699Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6eb94268-1054-4982-832b-831b223e834b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.277786423Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6eb94268-1054-4982-832b-831b223e834b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.278052258Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.278114379Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.27815655Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.278197281Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.281744654Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.286636526Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:47:35.420293037Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions","method":"POST","req_id":"0693bbbb-7ac9-41a1-9f8b-c2b67c2f24f3","remote_addr":"[fd00:1122:3344:101::a]:44943","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.302290962Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60996"}
{"msg":"region 6c0a25a6-2e05-4248-962c-ec9bd9afa4c4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.302532607Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.303217331Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.303227901Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions","method":"POST","req_id":"f4f19537-883b-4cda-9b0e-f7740a6413ee","remote_addr":"[fd00:1122:3344:101::a]:60996","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":721,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6c0a25a6-2e05-4248-962c-ec9bd9afa4c4","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.314673448Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6c0a25a6-2e05-4248-962c-ec9bd9afa4c4","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.314713849Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6c0a25a6-2e05-4248-962c-ec9bd9afa4c4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6c0a25a6-2e05-4248-962c-ec9bd9afa4c4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.378227301Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"region":"6c0a25a6-2e05-4248-962c-ec9bd9afa4c4","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6c0a25a6-2e05-4248-962c-ec9bd9afa4c4\" \"--data\" \"/data/regions/6c0a25a6-2e05-4248-962c-ec9bd9afa4c4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.380020688Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"region":"6c0a25a6-2e05-4248-962c-ec9bd9afa4c4","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.501403096Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"region":"6c0a25a6-2e05-4248-962c-ec9bd9afa4c4","component":"worker"}
{"msg":"region 6c0a25a6-2e05-4248-962c-ec9bd9afa4c4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.501452187Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.501542519Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6c0a25a6-2e05-4248-962c-ec9bd9afa4c4","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.502604691Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6c0a25a6-2e05-4248-962c-ec9bd9afa4c4","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.526096727Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.543172469Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6c0a25a6-2e05-4248-962c-ec9bd9afa4c4","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.549450189Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6c0a25a6-2e05-4248-962c-ec9bd9afa4c4","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.549740255Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.549798626Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.549841967Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.549884048Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.559204291Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.569390421Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:21.63272634Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions","method":"POST","req_id":"e541eb46-1eb0-4be7-a742-a0de164735ac","remote_addr":"[fd00:1122:3344:101::a]:60996","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.042102646Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions/6eb94268-1054-4982-832b-831b223e834b","method":"GET","req_id":"989e1ec7-b0cf-4936-9e9b-df09206a9a9f","remote_addr":"[fd00:1122:3344:101::a]:60996","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":627,"response_code":"200"}
{"msg":"path is \"/data/regions/6eb94268-1054-4982-832b-831b223e834b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.045224471Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"dataset is oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.054166406Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"snapshot is oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b@37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.062818085Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b\", \"37f3fb8c-b87c-443f-a1cf-79a4c951c424\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.062849945Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.073150128Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions/6eb94268-1054-4982-832b-831b223e834b/snapshots/37f3fb8c-b87c-443f-a1cf-79a4c951c424","method":"GET","req_id":"4b8c87c2-13b3-435b-8d5c-0a01b6305dd4","remote_addr":"[fd00:1122:3344:101::a]:60996","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27946,"response_code":"200"}
{"msg":"path is \"/data/regions/6eb94268-1054-4982-832b-831b223e834b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.07616923Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"dataset is oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.084495453Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"snapshot is oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b@37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.092080889Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b\", \"37f3fb8c-b87c-443f-a1cf-79a4c951c424\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.09210549Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 6eb94268-1054-4982-832b-831b223e834b-37f3fb8c-b87c-443f-a1cf-79a4c951c424 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.100265888Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"applying SMF actions for region 6eb94268-1054-4982-832b-831b223e834b running snapshot 37f3fb8c-b87c-443f-a1cf-79a4c951c424 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.100471063Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.100482953Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions/6eb94268-1054-4982-832b-831b223e834b/snapshots/37f3fb8c-b87c-443f-a1cf-79a4c951c424/run","method":"POST","req_id":"af8793da-b46f-465e-becd-6e32f90524ef","remote_addr":"[fd00:1122:3344:101::a]:60996","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24344,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-6eb94268-1054-4982-832b-831b223e834b-37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.10370795Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-6eb94268-1054-4982-832b-831b223e834b-37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.107791364Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-6eb94268-1054-4982-832b-831b223e834b-37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.123652642Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6eb94268-1054-4982-832b-831b223e834b/.zfs/snapshot/37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.123947898Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.12402167Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.12407744Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.124131092Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.124205573Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.133680579Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.138942718Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"worker"}
{"msg":"region 6eb94268-1054-4982-832b-831b223e834b running snapshot 37f3fb8c-b87c-443f-a1cf-79a4c951c424 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.138957448Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"path is \"/data/regions/6eb94268-1054-4982-832b-831b223e834b\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.267998815Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"dataset is oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.276104013Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"datafile"}
{"msg":"snapshot is oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b@37f3fb8c-b87c-443f-a1cf-79a4c951c424","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.284740921Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crucible/regions/6eb94268-1054-4982-832b-831b223e834b\", \"37f3fb8c-b87c-443f-a1cf-79a4c951c424\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.284776742Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-09T02:58:23.293301838Z","hostname":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","pid":26731,"uri":"/crucible/0/regions/6eb94268-1054-4982-832b-831b223e834b/snapshots/37f3fb8c-b87c-443f-a1cf-79a4c951c424/run","method":"POST","req_id":"1481de7f-dcb0-4f4a-b09b-b58ea7395daf","remote_addr":"[fd00:1122:3344:101::a]:60996","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":25353,"response_code":"200"}