[ Oct  9 21:34:19 Enabled. ]
[ Oct  9 21:34:19 Rereading configuration. ]
[ Oct  9 21:34:20 Rereading configuration. ]
[ Oct  9 21:34:22 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:34:22.274040159Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:34:22.275744924Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:34:22.275756004Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:34:22.296162968Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:34:22.403607476Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.747400966Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56916"}
{"msg":"region 4d4647da-819d-4348-8456-5640fd4e3884 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.767594126Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.77049039Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"uri":"/crucible/0/regions","method":"POST","req_id":"caf3c11c-6e3a-4c6c-b3d8-e85012085fcb","remote_addr":"[fd00:1122:3344:101::a]:56916","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":11337,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.771673017Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.780326046Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.780351116Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4d4647da-819d-4348-8456-5640fd4e3884\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4d4647da-819d-4348-8456-5640fd4e3884\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.825899894Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"region":"4d4647da-819d-4348-8456-5640fd4e3884","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4d4647da-819d-4348-8456-5640fd4e3884\" \"--data\" \"/data/regions/4d4647da-819d-4348-8456-5640fd4e3884\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.828020286Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"region":"4d4647da-819d-4348-8456-5640fd4e3884","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.971558102Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"region":"4d4647da-819d-4348-8456-5640fd4e3884","component":"worker"}
{"msg":"region 4d4647da-819d-4348-8456-5640fd4e3884 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.971619182Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.971695904Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4d4647da-819d-4348-8456-5640fd4e3884","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.97411084Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4d4647da-819d-4348-8456-5640fd4e3884","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.978102439Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.991828313Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4d4647da-819d-4348-8456-5640fd4e3884","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.994775777Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4d4647da-819d-4348-8456-5640fd4e3884","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.995309615Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.995371616Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.995413937Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.995454567Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:34.999395096Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:35.003963614Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:35:35.082457082Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"uri":"/crucible/0/regions","method":"POST","req_id":"4b34d23e-c4c6-4832-8be0-cd4c5cc33541","remote_addr":"[fd00:1122:3344:101::a]:56916","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.527851839Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58035"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.536246933Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"uri":"/crucible/0/regions/4d4647da-819d-4348-8456-5640fd4e3884","method":"GET","req_id":"41b8477a-0f15-4acc-b5ba-c140da897a53","remote_addr":"[fd00:1122:3344:101::a]:58035","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":8291,"response_code":"200"}
{"msg":"path is \"/data/regions/4d4647da-819d-4348-8456-5640fd4e3884\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.539169687Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"dataset is oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.548536026Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"snapshot is oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884@3fb5427b-19d4-45de-88b8-a6eba3e1464b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.556949541Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884\", \"3fb5427b-19d4-45de-88b8-a6eba3e1464b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.556986332Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.579166262Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"uri":"/crucible/0/regions/4d4647da-819d-4348-8456-5640fd4e3884/snapshots/3fb5427b-19d4-45de-88b8-a6eba3e1464b","method":"GET","req_id":"6b2d0de7-bd0b-40f6-a7be-b57116f41645","remote_addr":"[fd00:1122:3344:101::a]:58035","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":40024,"response_code":"200"}
{"msg":"path is \"/data/regions/4d4647da-819d-4348-8456-5640fd4e3884\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.583334454Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"dataset is oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.592614922Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"snapshot is oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884@3fb5427b-19d4-45de-88b8-a6eba3e1464b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.600385608Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884\", \"3fb5427b-19d4-45de-88b8-a6eba3e1464b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.600415018Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 4d4647da-819d-4348-8456-5640fd4e3884-3fb5427b-19d4-45de-88b8-a6eba3e1464b state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.610150633Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"applying SMF actions for region 4d4647da-819d-4348-8456-5640fd4e3884 running snapshot 3fb5427b-19d4-45de-88b8-a6eba3e1464b (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.610738812Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.610762082Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"uri":"/crucible/0/regions/4d4647da-819d-4348-8456-5640fd4e3884/snapshots/3fb5427b-19d4-45de-88b8-a6eba3e1464b/run","method":"POST","req_id":"0224072e-dbd3-4859-ad6c-297b59359912","remote_addr":"[fd00:1122:3344:101::a]:58035","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27455,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-4d4647da-819d-4348-8456-5640fd4e3884-3fb5427b-19d4-45de-88b8-a6eba3e1464b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.620754361Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-4d4647da-819d-4348-8456-5640fd4e3884-3fb5427b-19d4-45de-88b8-a6eba3e1464b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.627780366Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-4d4647da-819d-4348-8456-5640fd4e3884-3fb5427b-19d4-45de-88b8-a6eba3e1464b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.704738631Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4d4647da-819d-4348-8456-5640fd4e3884/.zfs/snapshot/3fb5427b-19d4-45de-88b8-a6eba3e1464b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.705231808Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.705279719Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.705322629Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.70536118Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.705410241Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.715394819Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.739043391Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"worker"}
{"msg":"region 4d4647da-819d-4348-8456-5640fd4e3884 running snapshot 3fb5427b-19d4-45de-88b8-a6eba3e1464b state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.739075642Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"path is \"/data/regions/4d4647da-819d-4348-8456-5640fd4e3884\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.976072959Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"dataset is oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.985134474Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"datafile"}
{"msg":"snapshot is oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884@3fb5427b-19d4-45de-88b8-a6eba3e1464b","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.992878579Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_b394feff-47e0-47be-bf31-05c4dccd4bdb/crucible/regions/4d4647da-819d-4348-8456-5640fd4e3884\", \"3fb5427b-19d4-45de-88b8-a6eba3e1464b\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:53.99292394Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T21:38:54.004088446Z","hostname":"oxz_crucible_b4eb92c2-3e1b-44b4-9335-852493565719","pid":21063,"uri":"/crucible/0/regions/4d4647da-819d-4348-8456-5640fd4e3884/snapshots/3fb5427b-19d4-45de-88b8-a6eba3e1464b/run","method":"POST","req_id":"2cbe4780-b674-4974-baee-dc9d6b2139b7","remote_addr":"[fd00:1122:3344:101::a]:58035","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28027,"response_code":"200"}