[ May  6 18:53:05 Enabled. ]
[ May  6 18:53:05 Rereading configuration. ]
[ May  6 18:53:10 Rereading configuration. ]
[ May  6 18:53:17 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_ad34c00c-d848-4310-92c7-641ce5863a7d/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_ad34c00c-d848-4310-92c7-641ce5863a7d/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.720335448Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.723937004Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.723950235Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.741091706Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.831942647Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.849094798Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57563"}
{"msg":"region ee3e5f20-1770-47da-8ed0-d007900044ff state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.912599633Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.915692567Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.915705047Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"uri":"/crucible/0/regions","method":"POST","req_id":"efdfc93d-bf21-4016-8b5e-0478d781e36e","remote_addr":"[fd00:1122:3344:101::a]:57563","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":9413,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_ad34c00c-d848-4310-92c7-641ce5863a7d/crucible/regions/ee3e5f20-1770-47da-8ed0-d007900044ff","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.926586778Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_ad34c00c-d848-4310-92c7-641ce5863a7d/crucible/regions/ee3e5f20-1770-47da-8ed0-d007900044ff","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.926617949Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ee3e5f20-1770-47da-8ed0-d007900044ff\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ee3e5f20-1770-47da-8ed0-d007900044ff\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.976731322Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"region":"ee3e5f20-1770-47da-8ed0-d007900044ff","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ee3e5f20-1770-47da-8ed0-d007900044ff\" \"--data\" \"/data/regions/ee3e5f20-1770-47da-8ed0-d007900044ff\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.978670598Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"region":"ee3e5f20-1770-47da-8ed0-d007900044ff","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.089216332Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"uri":"/crucible/0/regions","method":"POST","req_id":"9d059cb4-4177-4e00-b004-4bd7f01a8544","remote_addr":"[fd00:1122:3344:101::a]:57563","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.106139458Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"region":"ee3e5f20-1770-47da-8ed0-d007900044ff","component":"worker"}
{"msg":"region ee3e5f20-1770-47da-8ed0-d007900044ff state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.106163539Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.106284982Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ee3e5f20-1770-47da-8ed0-d007900044ff","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.106681651Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ee3e5f20-1770-47da-8ed0-d007900044ff","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.110439981Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.124337095Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ee3e5f20-1770-47da-8ed0-d007900044ff","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.127224734Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ee3e5f20-1770-47da-8ed0-d007900044ff","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.127941392Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.128075165Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.128126416Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.128167577Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.131775384Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.13579753Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.3532387Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"uri":"/crucible/0/regions","method":"POST","req_id":"0c03ed6a-1a6f-4ef2-84c5-1dc8e7b69b02","remote_addr":"[fd00:1122:3344:101::a]:57563","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.177879165Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61468"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.177992318Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"uri":"/crucible/0/regions/ee3e5f20-1770-47da-8ed0-d007900044ff","method":"GET","req_id":"a08f30cf-79eb-48f1-9530-a589bdea6ab2","remote_addr":"[fd00:1122:3344:101::a]:61468","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"path is \"/data/regions/ee3e5f20-1770-47da-8ed0-d007900044ff\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.178653454Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"datafile"}
{"msg":"dataset is oxp_ad34c00c-d848-4310-92c7-641ce5863a7d/crucible/regions/ee3e5f20-1770-47da-8ed0-d007900044ff","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.187636879Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"datafile"}
{"msg":"snapshot is oxp_ad34c00c-d848-4310-92c7-641ce5863a7d/crucible/regions/ee3e5f20-1770-47da-8ed0-d007900044ff@2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.195776054Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ad34c00c-d848-4310-92c7-641ce5863a7d/crucible/regions/ee3e5f20-1770-47da-8ed0-d007900044ff\", \"2741d331-a7ac-4ab3-9b92-e5769c6b8cab\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.195801945Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.212288641Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"uri":"/crucible/0/regions/ee3e5f20-1770-47da-8ed0-d007900044ff/snapshots/2741d331-a7ac-4ab3-9b92-e5769c6b8cab","method":"GET","req_id":"d9490396-b27a-4ab7-bd01-e5245328cba6","remote_addr":"[fd00:1122:3344:101::a]:61468","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":34139,"response_code":"200"}
{"msg":"path is \"/data/regions/ee3e5f20-1770-47da-8ed0-d007900044ff\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.212458075Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"datafile"}
{"msg":"dataset is oxp_ad34c00c-d848-4310-92c7-641ce5863a7d/crucible/regions/ee3e5f20-1770-47da-8ed0-d007900044ff","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.220286343Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"datafile"}
{"msg":"snapshot is oxp_ad34c00c-d848-4310-92c7-641ce5863a7d/crucible/regions/ee3e5f20-1770-47da-8ed0-d007900044ff@2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.228759426Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_ad34c00c-d848-4310-92c7-641ce5863a7d/crucible/regions/ee3e5f20-1770-47da-8ed0-d007900044ff\", \"2741d331-a7ac-4ab3-9b92-e5769c6b8cab\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.228792677Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot ee3e5f20-1770-47da-8ed0-d007900044ff-2741d331-a7ac-4ab3-9b92-e5769c6b8cab state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.237322102Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"datafile"}
{"msg":"applying SMF actions for region ee3e5f20-1770-47da-8ed0-d007900044ff running snapshot 2741d331-a7ac-4ab3-9b92-e5769c6b8cab (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.237871525Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.237929976Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"uri":"/crucible/0/regions/ee3e5f20-1770-47da-8ed0-d007900044ff/snapshots/2741d331-a7ac-4ab3-9b92-e5769c6b8cab/run","method":"POST","req_id":"7033202d-868b-4218-bb91-28615dee4245","remote_addr":"[fd00:1122:3344:101::a]:61468","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":25492,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-ee3e5f20-1770-47da-8ed0-d007900044ff-2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.249267338Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-ee3e5f20-1770-47da-8ed0-d007900044ff-2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.25515534Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-ee3e5f20-1770-47da-8ed0-d007900044ff-2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.266456651Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ee3e5f20-1770-47da-8ed0-d007900044ff/.zfs/snapshot/2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.266760868Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.26682906Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.266874541Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.266916082Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.266966543Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.270415186Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.274882933Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"worker"}
{"msg":"region ee3e5f20-1770-47da-8ed0-d007900044ff running snapshot 2741d331-a7ac-4ab3-9b92-e5769c6b8cab state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.274901754Z","hostname":"oxz_crucible_8e08297d-2b73-4c90-86c4-eb33b696f92f","pid":23011,"component":"datafile"}