[ May 29 21:50:15 Enabled. ]
[ May 29 21:50:15 Rereading configuration. ]
[ May 29 21:50:17 Rereading configuration. ]
[ May 29 21:50:19 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:20.050999222Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:20.051453947Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:20.051469437Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:20.082037747Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:20.251461734Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.681514558Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53958"}
{"msg":"region b6825550-bd82-4e6c-9ace-1362825bcd89 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.693389145Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.696401773Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.700782727Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"uri":"/crucible/0/regions","method":"POST","req_id":"7c4d4704-3295-4db0-b85b-add2a9cba10d","remote_addr":"[fd00:1122:3344:101::a]:53958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":9535,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible/regions/b6825550-bd82-4e6c-9ace-1362825bcd89","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.707859545Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible/regions/b6825550-bd82-4e6c-9ace-1362825bcd89","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.707926716Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b6825550-bd82-4e6c-9ace-1362825bcd89\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b6825550-bd82-4e6c-9ace-1362825bcd89\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.793464009Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"region":"b6825550-bd82-4e6c-9ace-1362825bcd89","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b6825550-bd82-4e6c-9ace-1362825bcd89\" \"--data\" \"/data/regions/b6825550-bd82-4e6c-9ace-1362825bcd89\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.795639456Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"region":"b6825550-bd82-4e6c-9ace-1362825bcd89","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.952388895Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"region":"b6825550-bd82-4e6c-9ace-1362825bcd89","component":"worker"}
{"msg":"region b6825550-bd82-4e6c-9ace-1362825bcd89 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.952474186Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.952732459Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b6825550-bd82-4e6c-9ace-1362825bcd89","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.953212875Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b6825550-bd82-4e6c-9ace-1362825bcd89","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.959965779Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.967708935Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b6825550-bd82-4e6c-9ace-1362825bcd89","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.970729283Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b6825550-bd82-4e6c-9ace-1362825bcd89","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.971607164Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.971708295Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.971754626Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.971801536Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.978236606Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.980052069Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"uri":"/crucible/0/regions","method":"POST","req_id":"1781cae2-ffb0-47ef-890e-685eb29841d3","remote_addr":"[fd00:1122:3344:101::a]:53958","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":899,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:50:39.984334402Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.250897967Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64081"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.253761433Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"uri":"/crucible/0/regions/b6825550-bd82-4e6c-9ace-1362825bcd89","method":"GET","req_id":"f9a4253e-f307-41d3-a2d8-299f06af6c3a","remote_addr":"[fd00:1122:3344:101::a]:64081","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":2837,"response_code":"200"}
{"msg":"path is \"/data/regions/b6825550-bd82-4e6c-9ace-1362825bcd89\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.254555423Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"datafile"}
{"msg":"dataset is oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible/regions/b6825550-bd82-4e6c-9ace-1362825bcd89","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.263566645Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"datafile"}
{"msg":"snapshot is oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible/regions/b6825550-bd82-4e6c-9ace-1362825bcd89@61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.272542227Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible/regions/b6825550-bd82-4e6c-9ace-1362825bcd89\", \"61a6668b-d30e-4ad8-b5b4-2668f0f727ec\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.272567557Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.285756521Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"uri":"/crucible/0/regions/b6825550-bd82-4e6c-9ace-1362825bcd89/snapshots/61a6668b-d30e-4ad8-b5b4-2668f0f727ec","method":"GET","req_id":"f2fdc573-955e-4110-aca9-9b2813b81420","remote_addr":"[fd00:1122:3344:101::a]:64081","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":31830,"response_code":"200"}
{"msg":"path is \"/data/regions/b6825550-bd82-4e6c-9ace-1362825bcd89\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.285928943Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"datafile"}
{"msg":"dataset is oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible/regions/b6825550-bd82-4e6c-9ace-1362825bcd89","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.294358888Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"datafile"}
{"msg":"snapshot is oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible/regions/b6825550-bd82-4e6c-9ace-1362825bcd89@61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.302421398Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_8f301520-a970-47c3-90c3-bd3006e5615c/crucible/regions/b6825550-bd82-4e6c-9ace-1362825bcd89\", \"61a6668b-d30e-4ad8-b5b4-2668f0f727ec\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.302450699Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot b6825550-bd82-4e6c-9ace-1362825bcd89-61a6668b-d30e-4ad8-b5b4-2668f0f727ec state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.31141009Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"datafile"}
{"msg":"applying SMF actions for region b6825550-bd82-4e6c-9ace-1362825bcd89 running snapshot 61a6668b-d30e-4ad8-b5b4-2668f0f727ec (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.311967337Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.312008087Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"uri":"/crucible/0/regions/b6825550-bd82-4e6c-9ace-1362825bcd89/snapshots/61a6668b-d30e-4ad8-b5b4-2668f0f727ec/run","method":"POST","req_id":"d159d123-ef7a-4079-9cc0-c1df7465601d","remote_addr":"[fd00:1122:3344:101::a]:64081","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26110,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-b6825550-bd82-4e6c-9ace-1362825bcd89-61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.32024912Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-b6825550-bd82-4e6c-9ace-1362825bcd89-61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.325935491Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-b6825550-bd82-4e6c-9ace-1362825bcd89-61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.336749705Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b6825550-bd82-4e6c-9ace-1362825bcd89/.zfs/snapshot/61a6668b-d30e-4ad8-b5b4-2668f0f727ec","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.337014998Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.337082959Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.33712718Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.33717065Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.337227901Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.340926937Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.345079469Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"worker"}
{"msg":"region b6825550-bd82-4e6c-9ace-1362825bcd89 running snapshot 61a6668b-d30e-4ad8-b5b4-2668f0f727ec state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-29T21:55:56.345093249Z","hostname":"oxz_crucible_d462ebe6-9d90-445d-bf8a-7c34338191d9","pid":23341,"component":"datafile"}