[ Jan  9 01:03:32 Enabled. ]
[ Jan  9 01:03:32 Rereading configuration. ]
[ Jan  9 01:03:34 Rereading configuration. ]
[ Jan  9 01:03:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:03:38.752136558Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:03:38.755383571Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:03:38.755395581Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:03:38.773588832Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:03:38.891300777Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:01.565249839Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46255"}
{"msg":"region 0ea978d6-db3f-44e4-803a-4a331c78f132 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:01.604139416Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:01.608488155Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:01.613764684Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"uri":"/crucible/0/regions","method":"POST","req_id":"3b6ac334-f7d7-46fa-a27f-b4e3689053eb","remote_addr":"[fd00:1122:3344:101::a]:46255","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19677,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:01.619623986Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:01.619680577Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0ea978d6-db3f-44e4-803a-4a331c78f132\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0ea978d6-db3f-44e4-803a-4a331c78f132\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:01.768427533Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"region":"0ea978d6-db3f-44e4-803a-4a331c78f132","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0ea978d6-db3f-44e4-803a-4a331c78f132\" \"--data\" \"/data/regions/0ea978d6-db3f-44e4-803a-4a331c78f132\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:01.770411548Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"region":"0ea978d6-db3f-44e4-803a-4a331c78f132","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:01.947101885Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"uri":"/crucible/0/regions","method":"POST","req_id":"3b729a1f-fe65-4458-abb7-ed22d03988d0","remote_addr":"[fd00:1122:3344:101::a]:46255","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.081076897Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"region":"0ea978d6-db3f-44e4-803a-4a331c78f132","component":"worker"}
{"msg":"region 0ea978d6-db3f-44e4-803a-4a331c78f132 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.081140319Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.081367124Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0ea978d6-db3f-44e4-803a-4a331c78f132","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.081819384Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0ea978d6-db3f-44e4-803a-4a331c78f132","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.09760589Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.133472859Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0ea978d6-db3f-44e4-803a-4a331c78f132","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.153470281Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0ea978d6-db3f-44e4-803a-4a331c78f132","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.154221058Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.154288459Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.15433062Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.154376631Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.177023012Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.201174167Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:04:02.206686501Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"uri":"/crucible/0/regions","method":"POST","req_id":"ff597dc2-741e-4bab-9fba-f3d106d86d54","remote_addr":"[fd00:1122:3344:101::a]:46255","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.056396004Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57610"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.060590659Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"uri":"/crucible/0/regions/0ea978d6-db3f-44e4-803a-4a331c78f132","method":"GET","req_id":"b6eaa11c-c6a2-4438-afe8-1ffe0a73a0cc","remote_addr":"[fd00:1122:3344:101::a]:57610","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":4128,"response_code":"200"}
{"msg":"path is \"/data/regions/0ea978d6-db3f-44e4-803a-4a331c78f132\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.065834557Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"dataset is oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.078765398Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"snapshot is oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132@976d8d11-2930-4c15-8236-b132904f582d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.087488255Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132\", \"976d8d11-2930-4c15-8236-b132904f582d\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.087519796Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.098391611Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"uri":"/crucible/0/regions/0ea978d6-db3f-44e4-803a-4a331c78f132/snapshots/976d8d11-2930-4c15-8236-b132904f582d","method":"GET","req_id":"47fee733-76ed-4964-b2b3-0ce34c7b456e","remote_addr":"[fd00:1122:3344:101::a]:57610","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":32593,"response_code":"200"}
{"msg":"path is \"/data/regions/0ea978d6-db3f-44e4-803a-4a331c78f132\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.101741527Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"dataset is oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.110686809Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"snapshot is oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132@976d8d11-2930-4c15-8236-b132904f582d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.119354974Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132\", \"976d8d11-2930-4c15-8236-b132904f582d\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.119391425Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 0ea978d6-db3f-44e4-803a-4a331c78f132-976d8d11-2930-4c15-8236-b132904f582d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.129829481Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"applying SMF actions for region 0ea978d6-db3f-44e4-803a-4a331c78f132 running snapshot 976d8d11-2930-4c15-8236-b132904f582d (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.130867054Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.130906955Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"uri":"/crucible/0/regions/0ea978d6-db3f-44e4-803a-4a331c78f132/snapshots/976d8d11-2930-4c15-8236-b132904f582d/run","method":"POST","req_id":"433c51b2-2a96-4919-8248-3f89e07522aa","remote_addr":"[fd00:1122:3344:101::a]:57610","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":29200,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-0ea978d6-db3f-44e4-803a-4a331c78f132-976d8d11-2930-4c15-8236-b132904f582d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.144120643Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-0ea978d6-db3f-44e4-803a-4a331c78f132-976d8d11-2930-4c15-8236-b132904f582d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.148564183Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-0ea978d6-db3f-44e4-803a-4a331c78f132-976d8d11-2930-4c15-8236-b132904f582d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.161108296Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0ea978d6-db3f-44e4-803a-4a331c78f132/.zfs/snapshot/976d8d11-2930-4c15-8236-b132904f582d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.161416863Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.161465775Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.161509045Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.161550826Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.161597578Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.165494175Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.169613548Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"worker"}
{"msg":"region 0ea978d6-db3f-44e4-803a-4a331c78f132 running snapshot 976d8d11-2930-4c15-8236-b132904f582d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.169623968Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"path is \"/data/regions/0ea978d6-db3f-44e4-803a-4a331c78f132\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.405447049Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"dataset is oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.415107887Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"datafile"}
{"msg":"snapshot is oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132@976d8d11-2930-4c15-8236-b132904f582d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.423903795Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible/regions/0ea978d6-db3f-44e4-803a-4a331c78f132\", \"976d8d11-2930-4c15-8236-b132904f582d\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.423954246Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-09T01:14:09.432342946Z","hostname":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","pid":26343,"uri":"/crucible/0/regions/0ea978d6-db3f-44e4-803a-4a331c78f132/snapshots/976d8d11-2930-4c15-8236-b132904f582d/run","method":"POST","req_id":"4eaf3a37-77bd-4204-b0c3-8d62664ee690","remote_addr":"[fd00:1122:3344:101::a]:57610","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":26942,"response_code":"200"}