01KX56JSRHBKF0TZZC3RBNTJVC: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KX56K5T4E0T0VF7H2PHE855J

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-07-10T05:18:41.423Zjob dependencies complete; ready to run (waiting for 14 m 19 s)
22026-07-10T05:19:44.608Zjob assigned to worker 01KX57DSS9AZYQNM5AT2D5W9YG [factory aws, i-0dfad18c93ad0cb00] (queued for 1 m 3 s)
32026-07-10T05:19:51.456Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-07-10T05:19:53.428Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-07-10T05:19:53.440Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-07-10T05:19:55.413Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-07-10T05:19:55.418Zdownloading input: /input/build/work/bins/crutest.gz
82026-07-10T05:19:57.449Zdownloaded input: /input/build/work/bins/crutest.gz
92026-07-10T05:19:57.453Zdownloading input: /input/build/work/bins/dsc.gz
102026-07-10T05:19:58.249Zdownloaded input: /input/build/work/bins/dsc.gz
112026-07-10T05:19:58.253Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-07-10T05:19:58.283Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-07-10T05:19:58.287Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-07-10T05:19:58.310Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-07-10T05:19:58.313Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-07-10T05:19:58.342Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-07-10T05:19:58.342Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-07-10T05:19:58.371Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-07-10T05:19:58.372Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-07-10T05:19:58.407Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-07-10T05:19:58.407Zdownloading input: /input/build/work/scripts/test_up.sh
222026-07-10T05:19:58.434Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-07-10T05:19:58.439Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-07-10T05:19:58.469Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-07-10T05:19:58.474Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-07-10T05:19:59.029Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-07-10T05:19:59.029Zstarting task 0: "setup"
282026-07-10T05:19:59.035Z++ uname -s
292026-07-10T05:19:59.039Z+ kern=SunOS
302026-07-10T05:19:59.039Z+ build_user=build
312026-07-10T05:19:59.039Z+ build_uid=12345
322026-07-10T05:19:59.039Z+ work_dir=/work
332026-07-10T05:19:59.039Z+ input_dir=/input
342026-07-10T05:19:59.039Z+ [[ 0 == 12345 ]]
352026-07-10T05:19:59.039Z+ case "$kern" in
362026-07-10T05:19:59.039Z+ groupadd -g 12345 build
372026-07-10T05:19:59.039Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-07-10T05:20:01.044Z+ zfs create -o mountpoint=/work rpool/work
392026-07-10T05:20:01.436Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-07-10T05:20:01.439Z+ home_fs=zfs
412026-07-10T05:20:01.439Z+ [[ zfs == autofs ]]
422026-07-10T05:20:01.439Z+ mkdir -p /home/build
432026-07-10T05:20:01.439Z+ chown build:build /home/build /work
442026-07-10T05:20:02.441Z+ chmod 0700 /home/build /work
452026-07-10T05:20:02.444Zprocess exited: duration 3412 ms, exit code 0
 
462026-07-10T05:20:02.450Zstarting task 1: "authentication"
472026-07-10T05:20:02.467Zprocess exited: duration 16 ms, exit code 0
 
482026-07-10T05:20:02.473Zstarting task 2: "build"
492026-07-10T05:20:02.476Z+ banner cores
502026-07-10T05:20:02.479Z
512026-07-10T05:20:02.479Z #### #### ##### ###### ####
522026-07-10T05:20:02.479Z # # # # # # # #
532026-07-10T05:20:02.479Z # # # # # ##### ####
542026-07-10T05:20:02.479Z # # # ##### # #
552026-07-10T05:20:02.479Z # # # # # # # # #
562026-07-10T05:20:02.479Z #### #### # # ###### ####
572026-07-10T05:20:02.479Z
582026-07-10T05:20:02.479Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-07-10T05:20:02.484Z+ banner unpack
602026-07-10T05:20:02.487Z
612026-07-10T05:20:02.487Z # # # # ##### ## #### # #
622026-07-10T05:20:02.487Z # # ## # # # # # # # # #
632026-07-10T05:20:02.487Z # # # # # # # # # # ####
642026-07-10T05:20:02.487Z # # # # # ##### ###### # # #
652026-07-10T05:20:02.487Z # # # ## # # # # # # #
662026-07-10T05:20:02.487Z #### # # # # # #### # #
672026-07-10T05:20:02.487Z
682026-07-10T05:20:02.487Z+ mkdir -p /var/tmp/bins
692026-07-10T05:20:02.487Z+ for t in "$input/bins/"*.gz
702026-07-10T05:20:02.490Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-07-10T05:20:02.490Z+ b=crucible-downstairs.gz
722026-07-10T05:20:02.490Z+ b=crucible-downstairs
732026-07-10T05:20:02.490Z+ gunzip
742026-07-10T05:20:05.596Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-07-10T05:20:05.599Z+ for t in "$input/bins/"*.gz
762026-07-10T05:20:05.599Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-07-10T05:20:05.599Z+ b=crucible-hammer.gz
782026-07-10T05:20:05.599Z+ b=crucible-hammer
792026-07-10T05:20:05.599Z+ gunzip
802026-07-10T05:20:08.556Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-07-10T05:20:08.560Z+ for t in "$input/bins/"*.gz
822026-07-10T05:20:08.560Z++ basename /input/build/work/bins/crutest.gz
832026-07-10T05:20:08.560Z+ b=crutest.gz
842026-07-10T05:20:08.560Z+ b=crutest
852026-07-10T05:20:08.560Z+ gunzip
862026-07-10T05:20:11.646Z+ chmod +x /var/tmp/bins/crutest
872026-07-10T05:20:11.650Z+ for t in "$input/bins/"*.gz
882026-07-10T05:20:11.650Z++ basename /input/build/work/bins/dsc.gz
892026-07-10T05:20:11.650Z+ b=dsc.gz
902026-07-10T05:20:11.650Z+ b=dsc
912026-07-10T05:20:11.650Z+ gunzip
922026-07-10T05:20:12.916Z+ chmod +x /var/tmp/bins/dsc
932026-07-10T05:20:12.919Z+ export BINDIR=/var/tmp/bins
942026-07-10T05:20:12.919Z+ BINDIR=/var/tmp/bins
952026-07-10T05:20:12.919Z+ jobpid=1088
962026-07-10T05:20:12.919Z+ echo 'Setup debug logging'
972026-07-10T05:20:12.919Z+ mkdir /tmp/debug
982026-07-10T05:20:12.919ZSetup debug logging
992026-07-10T05:20:12.919Z+ sleep 3600
1002026-07-10T05:20:12.922Z+ psrinfo -v
1012026-07-10T05:20:12.922Z+ df -h
1022026-07-10T05:20:12.927Z+ prstat -d d -mLc 1
1032026-07-10T05:20:12.930Z+ iostat -T d -xn 1
1042026-07-10T05:20:12.930Z+ mpstat -T d 1
1052026-07-10T05:20:12.930Z+ vmstat -T d -p 1
1062026-07-10T05:20:12.930Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-07-10T05:20:12.930Z+ banner test_up_2r_encrypted
1082026-07-10T05:20:12.930Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-07-10T05:20:12.933Z #####
1102026-07-10T05:20:12.933Z ##### ###### #### ##### # # ##### # # #####
1112026-07-10T05:20:12.933Z # # # # # # # # # # #
1122026-07-10T05:20:12.933Z # ##### #### # # # # # ##### # #
1132026-07-10T05:20:12.933Z # # # # # # ##### # #####
1142026-07-10T05:20:12.933Z # # # # # # # # # # #
1152026-07-10T05:20:12.933Z # ###### #### # ####### #### # ####### ####### # #
1162026-07-10T05:20:12.933Z
1172026-07-10T05:20:12.933Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-07-10T05:20:12.939Z/input/build/work
1192026-07-10T05:20:12.947ZUsing 2 region sets
1202026-07-10T05:20:12.947ZTurn off color for downstairs dump
1212026-07-10T05:20:12.959ZUpstairs using key: 24SRb6u81pwQ4fPP9DZHM1cupH4pN8vi/AgkIYrQGEs=
1222026-07-10T05:20:12.962Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-07-10T05:20:12.962ZCreating 6 downstairs regions
1242026-07-10T05:20:13.213ZStarting 6 downstairs
1252026-07-10T05:20:13.219Zdsc started at PID: 1136
1262026-07-10T05:20:18.307ZDisable automatic restart on all downstairs
1272026-07-10T05:20:18.320Z
1282026-07-10T05:20:18.320ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-07-10T05:20:18.324ZRunning test: span
1302026-07-10T05:20:22.435ZCompleted test: span
1312026-07-10T05:20:27.441ZRunning test: big
1322026-07-10T05:20:31.740ZCompleted test: big
1332026-07-10T05:20:36.747ZRunning test: dep
1342026-07-10T05:20:55.890ZCompleted test: dep
1352026-07-10T05:21:00.896ZRunning test: balloon
1362026-07-10T05:23:23.104ZCompleted test: balloon
1372026-07-10T05:23:28.109ZRunning test: deactivate
1382026-07-10T05:23:32.443ZCompleted test: deactivate
1392026-07-10T05:23:37.449ZRunning hammer
1402026-07-10T05:23:44.744ZRun repair tests
1412026-07-10T05:23:44.748Z/var/tmp/bins/crutest fill -g 31 -q --key 24SRb6u81pwQ4fPP9DZHM1cupH4pN8vi/AgkIYrQGEs= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-07-10T05:23:44.767Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.580730905Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"task":"crutest"}
1432026-07-10T05:23:44.771Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.581013632Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"task":"crutest"}
1442026-07-10T05:23:44.771Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.581659471Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"task":"crutest"}
1452026-07-10T05:23:44.771Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.581906275Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
1462026-07-10T05:23:44.771Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.58193561Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
1472026-07-10T05:23:44.771Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.581945918Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
1482026-07-10T05:23:44.771Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.581958835Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
1492026-07-10T05:23:44.771Z{"msg":"Crucible 4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796 has session id: 8a15ebcb-ef03-4caa-8476-3639eef182cc","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.582354121Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1502026-07-10T05:23:44.771Z{"msg":"Upstairs opts: Upstairs UUID: 4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.582395164Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1512026-07-10T05:23:44.771Z{"msg":"Crucible stats registered with UUID: 4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.582407501Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1522026-07-10T05:23:44.771Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:43.582417999Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1532026-07-10T05:23:44.772Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.582567701Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1542026-07-10T05:23:44.772Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.582584388Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1552026-07-10T05:23:44.772Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.582595866Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1562026-07-10T05:23:44.772Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583500017Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"task":"crutest"}
1572026-07-10T05:23:44.772Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583576962Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
1582026-07-10T05:23:44.772Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583598378Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
1592026-07-10T05:23:44.772Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583607907Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
1602026-07-10T05:23:44.772Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583620704Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
1612026-07-10T05:23:44.772Z{"msg":"Crucible 4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796 has session id: 548e442a-af44-43b8-afbf-d4e2cb7ecead","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583814078Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
1622026-07-10T05:23:44.772Z{"msg":"Upstairs opts: Upstairs UUID: 4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583844332Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
1632026-07-10T05:23:44.772Z{"msg":"Crucible stats registered with UUID: 4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583860299Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
1642026-07-10T05:23:44.772Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:43.583870448Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
1652026-07-10T05:23:44.772Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583880726Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
1662026-07-10T05:23:44.772Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583892043Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
1672026-07-10T05:23:44.772Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.583903091Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
1682026-07-10T05:23:44.772Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584066441Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"task":"crutest"}
1692026-07-10T05:23:44.772Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584090996Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
1702026-07-10T05:23:44.772Z{"msg":"4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584101174Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1712026-07-10T05:23:44.777Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584111652Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"0","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1722026-07-10T05:23:44.777Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584128119Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"1","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1732026-07-10T05:23:44.777Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584139257Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"2","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1742026-07-10T05:23:44.777Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584150465Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1752026-07-10T05:23:44.777Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584161633Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1762026-07-10T05:23:44.777Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584200026Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1772026-07-10T05:23:44.777Z{"msg":"ds_connection connected from Ok(127.0.0.1:65402)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584227801Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1782026-07-10T05:23:44.777Z{"msg":"ds_connection connected from Ok(127.0.0.1:33477)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584432932Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1792026-07-10T05:23:44.777Z{"msg":"ds_connection connected from Ok(127.0.0.1:57359)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584462447Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1802026-07-10T05:23:44.777Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584921721Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"2","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1812026-07-10T05:23:44.777Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584939317Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"1","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1822026-07-10T05:23:44.777Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.584973991Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"0","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1832026-07-10T05:23:44.777Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585113975Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1842026-07-10T05:23:44.777Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585133581Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1852026-07-10T05:23:44.778Z{"msg":"[0]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585221145Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1862026-07-10T05:23:44.778Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585232663Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1872026-07-10T05:23:44.778Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585243751Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1882026-07-10T05:23:44.778Z{"msg":"[1]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585255018Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1892026-07-10T05:23:44.778Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585266286Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1902026-07-10T05:23:44.778Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585277434Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1912026-07-10T05:23:44.778Z{"msg":"[2]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585288442Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1922026-07-10T05:23:44.778Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.58529932Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1932026-07-10T05:23:44.778Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585310548Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1942026-07-10T05:23:44.778Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585321236Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1952026-07-10T05:23:44.778Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585332514Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1962026-07-10T05:23:44.778Z{"msg":"Next flush: 3998","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585342982Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1972026-07-10T05:23:44.778Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.58535347Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1982026-07-10T05:23:44.778Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585369687Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
1992026-07-10T05:23:44.778Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585379405Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"0","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
2002026-07-10T05:23:44.778Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585393772Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"1","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
2012026-07-10T05:23:44.778Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.58540428Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"2","":"downstairs","session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
2022026-07-10T05:23:44.778Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585418958Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
2032026-07-10T05:23:44.778Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585436774Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
2042026-07-10T05:23:44.778Z{"msg":"4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796 is now active with session: 8a15ebcb-ef03-4caa-8476-3639eef182cc","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585447222Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
2052026-07-10T05:23:44.778Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585456941Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"8a15ebcb-ef03-4caa-8476-3639eef182cc"}
2062026-07-10T05:23:44.778Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585471088Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
2072026-07-10T05:23:44.778Z{"msg":"Activated sub_volume 4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585479776Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
2082026-07-10T05:23:44.778Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585488745Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
2092026-07-10T05:23:44.778Z{"msg":"4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585497313Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2102026-07-10T05:23:44.778Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585507061Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"0","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2112026-07-10T05:23:44.778Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585517449Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"1","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2122026-07-10T05:23:44.778Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585616351Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"2","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2132026-07-10T05:23:44.778Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585641776Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2142026-07-10T05:23:44.778Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585657753Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2152026-07-10T05:23:44.778Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585668791Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2162026-07-10T05:23:44.778Z{"msg":"ds_connection connected from Ok(127.0.0.1:34943)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585679829Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"0","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2172026-07-10T05:23:44.778Z{"msg":"ds_connection connected from Ok(127.0.0.1:46428)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585691367Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2182026-07-10T05:23:44.778Z{"msg":"ds_connection connected from Ok(127.0.0.1:52405)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585706434Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"1","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2192026-07-10T05:23:44.778Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.58594013Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"0","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2202026-07-10T05:23:44.778Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.585957007Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"2","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2212026-07-10T05:23:44.778Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586013706Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"1","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2222026-07-10T05:23:44.778Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586088243Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2232026-07-10T05:23:44.778Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.58610196Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2242026-07-10T05:23:44.779Z{"msg":"[0]R flush_numbers: [5320, 5319, 5317, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586146972Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2252026-07-10T05:23:44.779Z{"msg":"[0]R generation: [14, 14, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586158209Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2262026-07-10T05:23:44.779Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586168907Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2272026-07-10T05:23:44.779Z{"msg":"[1]R flush_numbers: [5320, 5319, 5317, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586185084Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2282026-07-10T05:23:44.779Z{"msg":"[1]R generation: [14, 14, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586195682Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2292026-07-10T05:23:44.779Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.58620641Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2302026-07-10T05:23:44.779Z{"msg":"[2]R flush_numbers: [5320, 5319, 5317, 5320, 5320]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586217328Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2312026-07-10T05:23:44.779Z{"msg":"[2]R generation: [14, 14, 13, 14, 14]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586232575Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2322026-07-10T05:23:44.779Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586243643Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2332026-07-10T05:23:44.779Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586254091Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2342026-07-10T05:23:44.779Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586264749Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2352026-07-10T05:23:44.779Z{"msg":"Next flush: 5321","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586274878Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2362026-07-10T05:23:44.779Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586285006Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"downstairs","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2372026-07-10T05:23:44.779Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586295844Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2382026-07-10T05:23:44.779Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586306032Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"0","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2392026-07-10T05:23:44.779Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.58631661Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"1","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2402026-07-10T05:23:44.779Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586327128Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"client":"2","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2412026-07-10T05:23:44.779Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586337486Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2422026-07-10T05:23:44.779Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586347864Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2432026-07-10T05:23:44.779Z{"msg":"4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796 is now active with session: 548e442a-af44-43b8-afbf-d4e2cb7ecead","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586359062Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2442026-07-10T05:23:44.779Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.58636883Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2452026-07-10T05:23:44.779Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586379318Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
2462026-07-10T05:23:44.779Z{Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2472026-07-10T05:23:44.779Z"msg":"Activated sub_volume 4f4c76e3-69c7-42e1-bc08-ee1fb2b6d796","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586392336Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
2482026-07-10T05:23:44.779Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586401644Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"task":"crutest"}
2492026-07-10T05:23:44.779Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.586415141Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"task":"crutest"}
2502026-07-10T05:23:44.797ZRead and Verify all blocks (0..100 range:false)
2512026-07-10T05:23:44.843Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"----------------------------------------------------------------
2522026-07-10T05:23:44.843Z2026-07-10T05:23:43.656577142Z" Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2532026-07-10T05:23:44.843Z,"States:hostname":" Activew-01KX57DSS9AZYQNM5AT2D5W9YG" Active, Active"
2542026-07-10T05:23:44.843Zpid":JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2552026-07-10T05:23:44.843Z1179,"task":"crutest"}
2562026-07-10T05:23:44.843Z 1002 Acked Read 50{ Done" Donemsg": Done" false
2572026-07-10T05:23:44.846ZCLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30 STATES DS:0 DS:1 DS:2 TOTAL
2582026-07-10T05:23:44.846Z Sent 0 0 0 0
2592026-07-10T05:23:44.846Z," Done 1 1 time 1 " 3
2602026-07-10T05:23:44.846Z:" Skipped 0 0 0 0
2612026-07-10T05:23:44.846Z2026-07-10T05:23:43.656612696Z"," Error 0 hostname" 0 :" 0 0
2622026-07-10T05:23:44.846Zw-01KX57DSS9AZYQNM5AT2D5W9YG","pid":Last Flush: 1179Some(JobId(1001)) ,"Some(JobId(1001)) Some(JobId(1001)) task
2632026-07-10T05:23:44.846Z"Downstairs last five completed::"crutest"}
2642026-07-10T05:23:44.846Z 1001 1000
2652026-07-10T05:23:44.846ZUpstairs last five completed: 1002 1001 1000
2662026-07-10T05:23:44.846Z----------------------------------------------------------------
2672026-07-10T05:23:44.846Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-07-10T05:23:44.846ZStates: Active Active Active
2692026-07-10T05:23:44.847ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-07-10T05:23:44.847Z 1002 Acked Read 50 Done Done Done false
2712026-07-10T05:23:44.847Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-07-10T05:23:44.847Z Sent 0 0 0 0
2732026-07-10T05:23:44.847Z Done 1 1 1 3
2742026-07-10T05:23:44.847Z Skipped 0 0 0 0
2752026-07-10T05:23:44.847Z Error 0 0 0 0
2762026-07-10T05:23:44.847ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-07-10T05:23:44.847ZDownstairs last five completed: 1001 1000
2782026-07-10T05:23:44.847ZUpstairs last five completed: 1002 1001 1000
2792026-07-10T05:23:44.847Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:43.656687651Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"task":"crutest"}
2802026-07-10T05:23:48.845Z----------------------------------------------------------------
2812026-07-10T05:23:48.845Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-07-10T05:23:48.845ZDownstairs last five completed: 1003 1002 1001 1000
2832026-07-10T05:23:48.845ZUpstairs last five completed: 1003 1002 1001 1000
2842026-07-10T05:23:48.849Z----------------------------------------------------------------
2852026-07-10T05:23:48.849Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-07-10T05:23:48.849ZDownstairs last five completed: 1003 1002 1001 1000
2872026-07-10T05:23:48.849ZUpstairs last five completed: 1003 1002 1001 1000
2882026-07-10T05:23:48.849Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.657568438Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"task":"crutest"}
2892026-07-10T05:23:48.849Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.6576142Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"task":"crutest"}
2902026-07-10T05:23:48.849Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:47.657626757Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
2912026-07-10T05:23:48.849Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:47.657636086Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179}
2922026-07-10T05:23:48.849Z{"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:47.657726439Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2932026-07-10T05:23:48.849Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:47.657740656Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2942026-07-10T05:23:48.849Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:47.657752274Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2952026-07-10T05:23:48.849Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.657763772Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1179,"":"io task","client":"2","":"downstairs","session_id":"548e442a-af44-43b8-afbf-d4e2cb7ecead"}
2962026-07-10T05:23:48.853ZRepair setup passed
2972026-07-10T05:23:48.856ZCopy the region for /var/tmp/test_up-build/8830
2982026-07-10T05:23:48.856Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2992026-07-10T05:23:48.859Z/var/tmp/bins/crutest fill -g 32 -q --key 24SRb6u81pwQ4fPP9DZHM1cupH4pN8vi/AgkIYrQGEs= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3002026-07-10T05:23:48.877Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.689795708Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
3012026-07-10T05:23:48.882Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.690143533Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
3022026-07-10T05:23:48.882Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.690897571Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
3032026-07-10T05:23:48.882Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69111966Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3042026-07-10T05:23:48.882Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.691145455Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3052026-07-10T05:23:48.882Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.691159143Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3062026-07-10T05:23:48.882Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69117195Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3072026-07-10T05:23:48.882Z{"msg":"Crucible df53d42f-3ef7-4b78-872c-ad581b470155 has session id: 0d0cb66a-117a-49cd-8795-7bda6f00b4a5","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.691585243Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3082026-07-10T05:23:48.882Z{"msg":"Upstairs opts: Upstairs UUID: df53d42f-3ef7-4b78-872c-ad581b470155, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.691623506Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3092026-07-10T05:23:48.882Z{"msg":"Crucible stats registered with UUID: df53d42f-3ef7-4b78-872c-ad581b470155","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.691639753Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3102026-07-10T05:23:48.882Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:47.69165405Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3112026-07-10T05:23:48.882Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.691786475Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3122026-07-10T05:23:48.882Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.691807781Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3132026-07-10T05:23:48.882Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.691818779Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3142026-07-10T05:23:48.882Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.692766802Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
3152026-07-10T05:23:48.882Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.692818092Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3162026-07-10T05:23:48.882Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.692843677Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3172026-07-10T05:23:48.882Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.692856805Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3182026-07-10T05:23:48.882Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.692868983Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3192026-07-10T05:23:48.883Z{"msg":"Crucible df53d42f-3ef7-4b78-872c-ad581b470155 has session id: ad42958a-c73a-440e-8f57-a627e649edc8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693189293Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3202026-07-10T05:23:48.883Z{"msg":"Upstairs opts: Upstairs UUID: df53d42f-3ef7-4b78-872c-ad581b470155, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693223076Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3212026-07-10T05:23:48.883Z{"msg":"Crucible stats registered with UUID: df53d42f-3ef7-4b78-872c-ad581b470155","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693238843Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3222026-07-10T05:23:48.883Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:47.693249281Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3232026-07-10T05:23:48.883Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.6932584Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3242026-07-10T05:23:48.883Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693274567Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3252026-07-10T05:23:48.883Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693285385Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3262026-07-10T05:23:48.883Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693386826Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
3272026-07-10T05:23:48.883Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693404332Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3282026-07-10T05:23:48.883Z{"msg":"df53d42f-3ef7-4b78-872c-ad581b470155 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693441725Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3292026-07-10T05:23:48.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693459092Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"0","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3302026-07-10T05:23:48.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69347011Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"1","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3312026-07-10T05:23:48.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693485427Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"2","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3322026-07-10T05:23:48.883Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693496075Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3332026-07-10T05:23:48.883Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693511312Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3342026-07-10T05:23:48.883Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69352244Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3352026-07-10T05:23:48.883Z{"msg":"ds_connection connected from Ok(127.0.0.1:59721)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693537617Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3362026-07-10T05:23:48.883Z{"msg":"ds_connection connected from Ok(127.0.0.1:57827)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693548555Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3372026-07-10T05:23:48.883Z{"msg":"ds_connection connected from Ok(127.0.0.1:35044)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.693563323Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3382026-07-10T05:23:48.888Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694258573Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"1","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3392026-07-10T05:23:48.888Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69432574Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"0","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3402026-07-10T05:23:48.888Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694359754Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"2","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3412026-07-10T05:23:48.888Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694510575Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3422026-07-10T05:23:48.889Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694550288Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3432026-07-10T05:23:48.889Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69464427Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3442026-07-10T05:23:48.889Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694663167Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3452026-07-10T05:23:48.889Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694674935Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3462026-07-10T05:23:48.889Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694685643Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3472026-07-10T05:23:48.889Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694696581Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3482026-07-10T05:23:48.889Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694715737Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3492026-07-10T05:23:48.889Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694730924Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3502026-07-10T05:23:48.889Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694741282Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3512026-07-10T05:23:48.889Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694756439Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3522026-07-10T05:23:48.889Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694766547Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3532026-07-10T05:23:48.889Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694781045Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3542026-07-10T05:23:48.889Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694791363Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3552026-07-10T05:23:48.889Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69480573Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3562026-07-10T05:23:48.889Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694816278Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3572026-07-10T05:23:48.889Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694829896Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"0","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3582026-07-10T05:23:48.889Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694840194Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"1","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3592026-07-10T05:23:48.889Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694854301Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"2","":"downstairs","session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3602026-07-10T05:23:48.889Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694864669Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3612026-07-10T05:23:48.889Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694874627Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3622026-07-10T05:23:48.889Z{"msg":"df53d42f-3ef7-4b78-872c-ad581b470155 is now active with session: 0d0cb66a-117a-49cd-8795-7bda6f00b4a5","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694885605Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3632026-07-10T05:23:48.889Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694895493Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"0d0cb66a-117a-49cd-8795-7bda6f00b4a5"}
3642026-07-10T05:23:48.889Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694905461Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3652026-07-10T05:23:48.889Z{"msg":"Activated sub_volume df53d42f-3ef7-4b78-872c-ad581b470155","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69491461Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3662026-07-10T05:23:48.889Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694923848Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
3672026-07-10T05:23:48.890Z{"msg":"df53d42f-3ef7-4b78-872c-ad581b470155 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694932147Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3682026-07-10T05:23:48.890Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694945714Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"0","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3692026-07-10T05:23:48.890Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.694994045Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"1","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3702026-07-10T05:23:48.890Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695052224Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"2","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3712026-07-10T05:23:48.890Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695085888Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3722026-07-10T05:23:48.890Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695097915Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3732026-07-10T05:23:48.890Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695109533Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3742026-07-10T05:23:48.890Z{"msg":"ds_connection connected from Ok(127.0.0.1:57599)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695121051Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3752026-07-10T05:23:48.890Z{"msg":"ds_connection connected from Ok(127.0.0.1:44954)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695132119Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3762026-07-10T05:23:48.890Z{"msg":"ds_connection connected from Ok(127.0.0.1:52411)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695143597Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3772026-07-10T05:23:48.890Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695373904Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"1","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3782026-07-10T05:23:48.890Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695418066Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"2","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3792026-07-10T05:23:48.890Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695442461Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"0","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3802026-07-10T05:23:48.890Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695545502Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3812026-07-10T05:23:48.890Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695563668Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3822026-07-10T05:23:48.890Z{"msg":"[0]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695574926Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3832026-07-10T05:23:48.890Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695585764Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3842026-07-10T05:23:48.890Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695600731Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3852026-07-10T05:23:48.890Z{"msg":"[1]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695611289Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3862026-07-10T05:23:48.890Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695625727Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3872026-07-10T05:23:48.890Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695636155Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3882026-07-10T05:23:48.890Z{"msg":"[2]R flush_numbers: [5321, 5321, 5321, 5321, 5321]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695650942Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3892026-07-10T05:23:48.890Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69566146Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3902026-07-10T05:23:48.890Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695676477Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3912026-07-10T05:23:48.890Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695687215Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3922026-07-10T05:23:48.890Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695701822Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3932026-07-10T05:23:48.890Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695711901Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3942026-07-10T05:23:48.890Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695725998Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"":"downstairs","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3952026-07-10T05:23:48.891Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695736556Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3962026-07-10T05:23:48.891Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695746184Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"0","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3972026-07-10T05:23:48.891Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695760581Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"1","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3982026-07-10T05:23:48.891Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69577052Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"client":"2","":"downstairs","session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
3992026-07-10T05:23:48.891Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695784957Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
4002026-07-10T05:23:48.891Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695795215Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
4012026-07-10T05:23:48.891Z{"msg":"df53d42f-3ef7-4b78-872c-ad581b470155 is now active with session: ad42958a-c73a-440e-8f57-a627e649edc8","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695809982Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
4022026-07-10T05:23:48.891Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.69581951Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"session_id":"ad42958a-c73a-440e-8f57-a627e649edc8"}
4032026-07-10T05:23:48.891Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695833508Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
4042026-07-10T05:23:48.891Z{"msg":"Activated sub_volume df53d42f-3ef7-4b78-872c-ad581b470155","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695842316Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
4052026-07-10T05:23:48.891Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.695855324Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
4062026-07-10T05:23:48.891ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4072026-07-10T05:23:48.891ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4082026-07-10T05:23:48.891Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.696649375Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
4092026-07-10T05:23:48.908ZRead and Verify all blocks (0..100 range:false)
4102026-07-10T05:23:48.954Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"----------------------------------------------------------------
4112026-07-10T05:23:48.954Zcrucible" Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4122026-07-10T05:23:48.954Z,"States:level": Active Active30 Active
4132026-07-10T05:23:48.954ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4142026-07-10T05:23:48.954Z 1002 Acked Read 50 Done Done Done false
4152026-07-10T05:23:48.954Z STATES DS:0 DS:1 DS:2 TOTAL
4162026-07-10T05:23:48.954Z," Sent time":" 0 0 0 0
4172026-07-10T05:23:48.954Z2026-07-10T05:23:47.766661604Z" Done 1 1 ," 1 3
4182026-07-10T05:23:48.954Zhostname Skipped ": 0 " 0 0 0
4192026-07-10T05:23:48.954Zw-01KX57DSS9AZYQNM5AT2D5W9YG" Error , 0 " 0 pid" 0 : 0
4202026-07-10T05:23:48.954Z1185Last Flush: Some(JobId(1001)) ,"Some(JobId(1001)) Some(JobId(1001)) task
4212026-07-10T05:23:48.955Z"Downstairs last five completed::"crutest"}
4222026-07-10T05:23:48.959Z 1001 1000
4232026-07-10T05:23:48.959Z{"msgUpstairs last five completed: ":" 1002 1001 1000
4242026-07-10T05:23:48.959ZCLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.766696308Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
4252026-07-10T05:23:48.959Z----------------------------------------------------------------
4262026-07-10T05:23:48.959Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4272026-07-10T05:23:48.959ZStates: Active Active Active
4282026-07-10T05:23:48.959ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4292026-07-10T05:23:48.959Z 1002 Acked Read 50 Done Done Done false
4302026-07-10T05:23:48.959Z STATES DS:0 DS:1 DS:2 TOTAL
4312026-07-10T05:23:48.959Z Sent 0 0 0 0
4322026-07-10T05:23:48.959Z Done 1 1 1 3
4332026-07-10T05:23:48.959Z Skipped 0 0 0 0
4342026-07-10T05:23:48.959Z Error 0 0 0 0
4352026-07-10T05:23:48.959ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4362026-07-10T05:23:48.959ZDownstairs last five completed: 1001 1000
4372026-07-10T05:23:48.959ZUpstairs last five completed: 1002 1001 1000
4382026-07-10T05:23:48.959Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:47.766783811Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
4392026-07-10T05:23:52.956Z----------------------------------------------------------------
4402026-07-10T05:23:52.956Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4412026-07-10T05:23:52.956ZDownstairs last five completed: 1003 1002 1001 1000
4422026-07-10T05:23:52.956ZUpstairs last five completed: 1003 1002 1001 1000
4432026-07-10T05:23:52.970Z----------------------------------------------------------------
4442026-07-10T05:23:52.970Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4452026-07-10T05:23:52.970ZDownstairs last five completed: 1003 1002 1001 1000
4462026-07-10T05:23:52.970ZUpstairs last five completed: 1003 1002 1001 1000
4472026-07-10T05:23:52.970Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.768332283Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
4482026-07-10T05:23:52.970Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.768375985Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185,"task":"crutest"}
4492026-07-10T05:23:52.970Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:51.768388193Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1185}
4502026-07-10T05:23:52.970ZRepair part 1 passed
4512026-07-10T05:23:52.970Z
4522026-07-10T05:23:52.970ZKill the current downstairs
4532026-07-10T05:23:53.005ZDownstairs 2 stopped
4542026-07-10T05:23:53.005Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4552026-07-10T05:23:53.009ZNow put back the original so we have a mismatch
4562026-07-10T05:23:53.009Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4572026-07-10T05:23:53.009ZRestart downstairs with old directory
4582026-07-10T05:23:53.021Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4592026-07-10T05:23:53.034ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4602026-07-10T05:23:53.037Z 0 000-009 32 32 31 3999 3999 3998 F F F <---
4612026-07-10T05:23:53.037Z 1 010-019 32 32 31 3999 3999 3998 F F F <---
4622026-07-10T05:23:53.037Z 2 020-029 32 32 31 3999 3999 3998 F F F <---
4632026-07-10T05:23:53.037Z 3 030-039 32 32 31 3999 3999 3998 F F F <---
4642026-07-10T05:23:53.037Z 4 040-049 32 32 31 3999 3999 3998 F F F <---
4652026-07-10T05:23:53.037ZMax gen: 32, Max flush: 3999
4662026-07-10T05:23:53.037ZError: Difference in extent metadata found!
4672026-07-10T05:23:53.037Zdump test found error as expected
4682026-07-10T05:23:53.037Z
4692026-07-10T05:23:53.037Z
4702026-07-10T05:23:53.037Z/var/tmp/bins/crutest verify --range -g 33 -q --key 24SRb6u81pwQ4fPP9DZHM1cupH4pN8vi/AgkIYrQGEs= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4712026-07-10T05:23:53.055Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.867270529Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
4722026-07-10T05:23:53.059Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.867626612Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
4732026-07-10T05:23:53.059Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.868433321Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
4742026-07-10T05:23:53.059Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.868663548Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
4752026-07-10T05:23:53.059Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.868688794Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
4762026-07-10T05:23:53.059Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.868702461Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
4772026-07-10T05:23:53.059Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.868714899Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
4782026-07-10T05:23:53.059Z{"msg":"Crucible 28eeed3c-99c8-4442-9db2-b1ec990035ee has session id: 1e714a28-de4b-4ad7-852c-4283a820b313","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.869113854Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
4792026-07-10T05:23:53.059Z{"msg":"Upstairs opts: Upstairs UUID: 28eeed3c-99c8-4442-9db2-b1ec990035ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.86913335Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
4802026-07-10T05:23:53.059Z{"msg":"Crucible stats registered with UUID: 28eeed3c-99c8-4442-9db2-b1ec990035ee","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.869145198Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
4812026-07-10T05:23:53.059Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:51.869155456Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
4822026-07-10T05:23:53.059Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.869331363Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
4832026-07-10T05:23:53.059Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.8693483Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
4842026-07-10T05:23:53.059Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.869360038Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
4852026-07-10T05:23:53.059Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870277996Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
4862026-07-10T05:23:53.059Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870334536Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
4872026-07-10T05:23:53.059Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7bedda9691592e7e5a5decf56da8eeb32141ddb0\",\n git_commit_timestamp: \"2026-07-10T05:03:43.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"2d8144b7880597b6e6d3dfd63a9a9efae3f533d3\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870355062Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
4882026-07-10T05:23:53.059Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.87036438Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
4892026-07-10T05:23:53.059Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870376368Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
4902026-07-10T05:23:53.059Z{"msg":"Crucible 28eeed3c-99c8-4442-9db2-b1ec990035ee has session id: 4a49b2de-116e-4160-9e3b-743993368ef0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870540057Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
4912026-07-10T05:23:53.059Z{"msg":"Upstairs opts: Upstairs UUID: 28eeed3c-99c8-4442-9db2-b1ec990035ee, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870562703Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
4922026-07-10T05:23:53.059Z{"msg":"Crucible stats registered with UUID: 28eeed3c-99c8-4442-9db2-b1ec990035ee","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870572901Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
4932026-07-10T05:23:53.059Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:51.870582909Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
4942026-07-10T05:23:53.059Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870592098Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
4952026-07-10T05:23:53.059Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870607285Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
4962026-07-10T05:23:53.063Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870664564Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
4972026-07-10T05:23:53.063Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870835242Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
4982026-07-10T05:23:53.063Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870849209Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
4992026-07-10T05:23:53.063Z{"msg":"28eeed3c-99c8-4442-9db2-b1ec990035ee active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870884143Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5002026-07-10T05:23:53.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870910778Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"0","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5012026-07-10T05:23:53.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870921816Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"1","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5022026-07-10T05:23:53.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870932454Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5032026-07-10T05:23:53.063Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870942552Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5042026-07-10T05:23:53.063Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.8709534Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5052026-07-10T05:23:53.063Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870968227Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5062026-07-10T05:23:53.063Z{"msg":"ds_connection connected from Ok(127.0.0.1:39930)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870979885Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5072026-07-10T05:23:53.063Z{"msg":"ds_connection connected from Ok(127.0.0.1:49946)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.870991413Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5082026-07-10T05:23:53.063Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:51.871067888Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5092026-07-10T05:23:53.063Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:51.871096753Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5102026-07-10T05:23:53.063Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:23:51.87111577Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5112026-07-10T05:23:53.063Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.871125648Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5122026-07-10T05:23:53.063Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.871146644Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5132026-07-10T05:23:53.063Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.871602748Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"1","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5142026-07-10T05:23:53.063Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.871628224Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"0","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5152026-07-10T05:23:53.063Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.871772597Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5162026-07-10T05:23:53.063Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:23:51.871787134Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5172026-07-10T05:24:03.063Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.872711461Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5182026-07-10T05:24:03.063Z{"msg":"ds_connection connected from Ok(127.0.0.1:60064)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.87276884Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5192026-07-10T05:24:03.068Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.873875723Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5202026-07-10T05:24:03.068Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874129425Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5212026-07-10T05:24:03.068Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874145652Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5222026-07-10T05:24:03.068Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.87415642Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5232026-07-10T05:24:03.068Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874171538Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5242026-07-10T05:24:03.068Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874181896Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5252026-07-10T05:24:03.068Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874196883Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5262026-07-10T05:24:03.068Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874207511Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5272026-07-10T05:24:03.068Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874221988Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5282026-07-10T05:24:03.068Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874232336Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5292026-07-10T05:24:03.068Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874242294Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5302026-07-10T05:24:03.068Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874252912Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5312026-07-10T05:24:03.068Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.87426687Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5322026-07-10T05:24:03.068Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874281617Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5332026-07-10T05:24:03.069Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874301463Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5342026-07-10T05:24:03.069Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.87431691Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5352026-07-10T05:24:03.069Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874331188Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5362026-07-10T05:24:03.069Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874350154Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5372026-07-10T05:24:03.069Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874361362Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5382026-07-10T05:24:03.069Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874375629Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5392026-07-10T05:24:03.069Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874395536Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5402026-07-10T05:24:03.069Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874410843Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5412026-07-10T05:24:03.069Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.87442528Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5422026-07-10T05:24:03.069Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874443827Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5432026-07-10T05:24:03.069Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874459284Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5442026-07-10T05:24:03.069Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874473621Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5452026-07-10T05:24:03.069Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874491858Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5462026-07-10T05:24:03.069Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874503086Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"mend","":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5472026-07-10T05:24:03.069Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.87453056Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"0","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5482026-07-10T05:24:03.069Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874546118Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"1","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5492026-07-10T05:24:03.069Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874556686Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5502026-07-10T05:24:03.069Z{"msg":"Full repair list: {ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.87458711Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5512026-07-10T05:24:03.069Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874683792Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5522026-07-10T05:24:03.069Z{"msg":"starting reconciliation 7a07f14e-fdc1-4c41-bbe2-f54da5b734a4: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.87469579Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5532026-07-10T05:24:03.069Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.874861888Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5542026-07-10T05:24:03.073Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.883735658Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5552026-07-10T05:24:03.076Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.885771447Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5562026-07-10T05:24:03.079Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.887673471Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5572026-07-10T05:24:03.079Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.88954761Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5582026-07-10T05:24:03.083Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891287415Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5592026-07-10T05:24:03.083Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.89131258Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"0","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5602026-07-10T05:24:03.083Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891323608Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"1","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5612026-07-10T05:24:03.083Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891338755Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5622026-07-10T05:24:03.083Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891349103Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5632026-07-10T05:24:03.083Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891363441Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5642026-07-10T05:24:03.083Z{"msg":"28eeed3c-99c8-4442-9db2-b1ec990035ee is now active with session: 1e714a28-de4b-4ad7-852c-4283a820b313","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891374448Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
5652026-07-10T05:24:03.083Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891388616Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
5662026-07-10T05:24:03.083Z{"msg":"Activated sub_volume 28eeed3c-99c8-4442-9db2-b1ec990035ee","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891408142Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
5672026-07-10T05:24:03.083Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.89147516Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
5682026-07-10T05:24:03.083Z{"msg":"28eeed3c-99c8-4442-9db2-b1ec990035ee active request set","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891489387Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5692026-07-10T05:24:03.083Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891498945Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"0","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5702026-07-10T05:24:03.083Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891509483Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"1","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5712026-07-10T05:24:03.083Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.89152413Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5722026-07-10T05:24:03.083Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891538448Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5732026-07-10T05:24:03.083Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891553005Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5742026-07-10T05:24:03.083Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891563663Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5752026-07-10T05:24:03.083Z{"msg":"ds_connection connected from Ok(127.0.0.1:45957)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.8915785Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5762026-07-10T05:24:03.083Z{"msg":"ds_connection connected from Ok(127.0.0.1:34779)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891594657Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5772026-07-10T05:24:03.083Z{"msg":"ds_connection connected from Ok(127.0.0.1:52773)","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.891605435Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5782026-07-10T05:24:03.083Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892030046Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"0","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5792026-07-10T05:24:03.084Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892053141Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"1","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5802026-07-10T05:24:03.084Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892064379Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5812026-07-10T05:24:03.084Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892198414Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5822026-07-10T05:24:03.084Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892209082Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5832026-07-10T05:24:03.084Z{"msg":"[0]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892273Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5842026-07-10T05:24:03.084Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892285048Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5852026-07-10T05:24:03.084Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892295806Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5862026-07-10T05:24:03.084Z{"msg":"[1]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892306894Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5872026-07-10T05:24:03.084Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892317662Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5882026-07-10T05:24:03.084Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.89232807Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5892026-07-10T05:24:03.084Z{"msg":"[2]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892339188Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5902026-07-10T05:24:03.084Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892349756Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5912026-07-10T05:24:03.084Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892360584Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5922026-07-10T05:24:03.084Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892371322Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5932026-07-10T05:24:03.084Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.89238271Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5942026-07-10T05:24:03.084Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892392648Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5952026-07-10T05:24:03.084Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892403126Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"downstairs","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5962026-07-10T05:24:03.084Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892413914Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5972026-07-10T05:24:03.084Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892423942Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"0","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5982026-07-10T05:24:03.084Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.89243431Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"1","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
5992026-07-10T05:24:03.084Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892445068Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"client":"2","":"downstairs","session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
6002026-07-10T05:24:03.084Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892455526Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
6012026-07-10T05:24:03.084Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892465834Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
6022026-07-10T05:24:03.084Z{"msg":"28eeed3c-99c8-4442-9db2-b1ec990035ee is now active with session: 4a49b2de-116e-4160-9e3b-743993368ef0","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892476762Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
6032026-07-10T05:24:03.084Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.89248636Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"session_id":"4a49b2de-116e-4160-9e3b-743993368ef0"}
6042026-07-10T05:24:03.084Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892496139Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
6052026-07-10T05:24:03.084Z{"msg":"Activated sub_volume 28eeed3c-99c8-4442-9db2-b1ec990035ee","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892505407Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
6062026-07-10T05:24:03.084Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.892514125Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
6072026-07-10T05:24:03.084ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6082026-07-10T05:24:03.084ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6092026-07-10T05:24:03.084ZRead and Verify all blocks (0..100 range:true)
6102026-07-10T05:24:03.130Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.940169468Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
6112026-07-10T05:24:03.130Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.940190504Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
6122026-07-10T05:24:03.133Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.940232596Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
6132026-07-10T05:24:03.133Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.940247063Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
6142026-07-10T05:24:03.133Z----------------------------------------------------------------
6152026-07-10T05:24:03.133Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6162026-07-10T05:24:03.133ZStates: Active Active Active
6172026-07-10T05:24:03.133ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6182026-07-10T05:24:03.133Z 1000 Acked Read 50 Done Done Done false
6192026-07-10T05:24:03.133Z STATES DS:0 DS:1 DS:2 TOTAL
6202026-07-10T05:24:03.133Z Sent 0 0 0 0
6212026-07-10T05:24:03.133Z Done 1 1 1 3
6222026-07-10T05:24:03.133Z Skipped 0 0 0 0
6232026-07-10T05:24:03.133Z Error 0 0 0 0
6242026-07-10T05:24:03.133ZLast Flush: None None None
6252026-07-10T05:24:03.133ZDownstairs last five completed:
6262026-07-10T05:24:03.133ZUpstairs last five completed: 1000
6272026-07-10T05:24:03.134Z----------------------------------------------------------------
6282026-07-10T05:24:03.134Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6292026-07-10T05:24:03.134ZStates: Active Active Active
6302026-07-10T05:24:03.134ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6312026-07-10T05:24:03.134Z 1000 Acked Read 50 Done Done Done false
6322026-07-10T05:24:03.134Z STATES DS:0 DS:1 DS:2 TOTAL
6332026-07-10T05:24:03.134Z Sent 0 0 0 0
6342026-07-10T05:24:03.134Z Done 1 1 1 3
6352026-07-10T05:24:03.134Z Skipped 0 0 0 0
6362026-07-10T05:24:03.134Z Error 0 0 0 0
6372026-07-10T05:24:03.134ZLast Flush: None None None
6382026-07-10T05:24:03.134ZDownstairs last five completed:
6392026-07-10T05:24:03.134ZUpstairs last five completed: 1000
6402026-07-10T05:24:03.134Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:01.940378029Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
6412026-07-10T05:24:07.132Z----------------------------------------------------------------
6422026-07-10T05:24:07.132Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6432026-07-10T05:24:07.132ZDownstairs last five completed: 1001 1000
6442026-07-10T05:24:07.132ZUpstairs last five completed: 1001 1000
6452026-07-10T05:24:07.136Z----------------------------------------------------------------
6462026-07-10T05:24:07.136Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6472026-07-10T05:24:07.136ZDownstairs last five completed: 1001 1000
6482026-07-10T05:24:07.136ZUpstairs last five completed: 1001 1000
6492026-07-10T05:24:07.136Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:05.941491522Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
6502026-07-10T05:24:07.136Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:05.941519577Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"task":"crutest"}
6512026-07-10T05:24:07.136Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:05.941530465Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193}
6522026-07-10T05:24:07.136Z{"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:05.941681497Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
6532026-07-10T05:24:07.136Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:05.941701643Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
6542026-07-10T05:24:07.136Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-07-10T05:24:05.9417167Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
6552026-07-10T05:24:07.136Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-07-10T05:24:05.941729808Z","hostname":"w-01KX57DSS9AZYQNM5AT2D5W9YG","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"1e714a28-de4b-4ad7-852c-4283a820b313"}
6562026-07-10T05:24:07.139ZRepair part 2 passed
6572026-07-10T05:24:07.139Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6582026-07-10T05:24:07.150ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6592026-07-10T05:24:07.150Z 0 000-009 32 32 32 3999 3999 3999 F F F
6602026-07-10T05:24:07.154Z 1 010-019 32 32 32 3999 3999 3999 F F F
6612026-07-10T05:24:07.154Z 2 020-029 32 32 32 3999 3999 3999 F F F
6622026-07-10T05:24:07.154Z 3 030-039 32 32 32 3999 3999 3999 F F F
6632026-07-10T05:24:07.154Z 4 040-049 32 32 32 3999 3999 3999 F F F
6642026-07-10T05:24:07.154ZMax gen: 32, Max flush: 3999
6652026-07-10T05:24:07.154Zdump test passed
6662026-07-10T05:24:07.154Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6672026-07-10T05:24:07.164Z Extent 1
6682026-07-10T05:24:07.167ZGEN 32 32 32
6692026-07-10T05:24:07.167ZFLUSH_ID 3999 3999 3999
6702026-07-10T05:24:07.167ZDIRTY
6712026-07-10T05:24:07.167Z
6722026-07-10T05:24:07.167ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6732026-07-10T05:24:07.167Z010 A A A A A A
6742026-07-10T05:24:07.167Z011 A A A A A A
6752026-07-10T05:24:07.167Z012 A A A A A A
6762026-07-10T05:24:07.167Z013 A A A A A A
6772026-07-10T05:24:07.171Z014 A A A A A A
6782026-07-10T05:24:07.171Z015 A A A A A A
6792026-07-10T05:24:07.171Z016 A A A A A A
6802026-07-10T05:24:07.171Z017 A A A A A A
6812026-07-10T05:24:07.171Z018 A A A A A A
6822026-07-10T05:24:07.171Z019 A A A A A A
6832026-07-10T05:24:07.174Zdump extent test passed
6842026-07-10T05:24:07.174Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6852026-07-10T05:24:07.182ZExtent 2 Block in extent 0 Actual block 20
6862026-07-10T05:24:07.182Z
6872026-07-10T05:24:07.186Z DATA SHA256 VER
6882026-07-10T05:24:07.186Z------ ---------------------------------------------------------------- ---
6892026-07-10T05:24:07.186Z 0 4868b7768b99fbe8fe53ef3f78f353ae6211a5a90ce04ee1100c39f3c34c89af A
6902026-07-10T05:24:07.186Z 1 4868b7768b99fbe8fe53ef3f78f353ae6211a5a90ce04ee1100c39f3c34c89af A
6912026-07-10T05:24:07.186Z 2 4868b7768b99fbe8fe53ef3f78f353ae6211a5a90ce04ee1100c39f3c34c89af A
6922026-07-10T05:24:07.186Z
6932026-07-10T05:24:07.186ZNONCES 0 1 2 DIFF
6942026-07-10T05:24:07.186Z------ ------------------------ ------------------------ ------------------------ -----
6952026-07-10T05:24:07.186Z 0 13c05f07376a0523be32c28d 13c05f07376a0523be32c28d 13c05f07376a0523be32c28d
6962026-07-10T05:24:07.186Z
6972026-07-10T05:24:07.186Z TAGS 0 1 2 DIFF
6982026-07-10T05:24:07.186Z------ -------------------------------- -------------------------------- -------------------------------- -----
6992026-07-10T05:24:07.186Z 0 879f6ca763b90b6411633c8f4d8a5c6a 879f6ca763b90b6411633c8f4d8a5c6a 879f6ca763b90b6411633c8f4d8a5c6a
7002026-07-10T05:24:07.186Z
7012026-07-10T05:24:07.186ZHASHES 0 1 2 DIFF
7022026-07-10T05:24:07.186Z------ ---------------- ---------------- ---------------- -----
7032026-07-10T05:24:07.186Z
7042026-07-10T05:24:07.190Zdump block test passed
7052026-07-10T05:24:07.190ZInitial upstairs tests have completed, stopping all downstairs
7062026-07-10T05:24:12.207ZCreating 7 larger downstairs regions
7072026-07-10T05:24:12.429ZStarting 7 downstairs
7082026-07-10T05:24:17.435Zdsc restarted at PID: 1209
7092026-07-10T05:24:17.537ZNow do the replace-reconcile test
7102026-07-10T05:24:17.550ZUsing 8870 for the replacement port
7112026-07-10T05:27:57.134ZCompleted test: replace-reconcile
7122026-07-10T05:27:57.137ZNow do the replace-before-active test
7132026-07-10T05:31:49.556ZCompleted test: replace-before-active
7142026-07-10T05:31:49.560ZAll tests have completed, stopping all downstairs
7152026-07-10T05:31:49.575Z
7162026-07-10T05:31:49.575Zreal 11:36.506122219
7172026-07-10T05:31:49.575Zuser 16:22.048803671
7182026-07-10T05:31:49.575Zsys 11.943234931
7192026-07-10T05:31:49.575Ztrap 0.431021506
7202026-07-10T05:31:49.575Ztflt 0.045001028
7212026-07-10T05:31:49.575Zdflt 0.018324246
7222026-07-10T05:31:49.575Zkflt 0.001082040
7232026-07-10T05:31:49.575Zlock 15:44:54.815635129
7242026-07-10T05:31:49.575Zslp 49:44.521957654
7252026-07-10T05:31:49.575Zlat 1:51.746747555
7262026-07-10T05:31:49.575Zstop 0.059633650
7272026-07-10T05:31:49.575Z+ echo 'test-up-2region-encrypted ends'
7282026-07-10T05:31:49.578Z
7292026-07-10T05:31:49.578ZAll Tests have passed
7302026-07-10T05:31:49.578Z11:37 Test duration
7312026-07-10T05:31:49.578Ztest-up-2region-encrypted ends
7322026-07-10T05:31:54.577Zprocess exited: duration 706965 ms, exit code 0
7332026-07-10T05:31:54.578Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7342026-07-10T05:32:54.599Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7352026-07-10T05:32:54.599Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7362026-07-10T05:32:54.611Zfound 17 output files
7372026-07-10T05:32:54.612Zuploading: /tmp/test_up-build/dsc-out.txt (59018 bytes)
7382026-07-10T05:32:55.625Zuploaded: /tmp/test_up-build/dsc-out.txt
7392026-07-10T05:32:55.625Zuploading: /tmp/test_up-build/test_up_out.txt (23129918 bytes)
7402026-07-10T05:32:56.774Zuploaded: /tmp/test_up-build/test_up_out.txt
7412026-07-10T05:32:56.774Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550947 bytes)
7422026-07-10T05:32:57.800Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7432026-07-10T05:32:57.800Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548255 bytes)
7442026-07-10T05:32:58.822Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7452026-07-10T05:32:58.822Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978660 bytes)
7462026-07-10T05:32:59.844Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7472026-07-10T05:32:59.844Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543059 bytes)
7482026-07-10T05:33:00.865Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7492026-07-10T05:33:00.865Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2493 bytes)
7502026-07-10T05:33:01.876Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7512026-07-10T05:33:01.876Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (47333 bytes)
7522026-07-10T05:33:02.889Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7532026-07-10T05:33:02.889Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3592791 bytes)
7542026-07-10T05:33:03.926Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7552026-07-10T05:33:03.926Zuploading: /tmp/debug/df.txt (1270 bytes)
7562026-07-10T05:33:04.938Zuploaded: /tmp/debug/df.txt
7572026-07-10T05:33:04.938Zuploading: /tmp/debug/dtrace.txt (1030178 bytes)
7582026-07-10T05:33:05.963Zuploaded: /tmp/debug/dtrace.txt
7592026-07-10T05:33:05.963Zuploading: /tmp/debug/iostat.txt (226577 bytes)
7602026-07-10T05:33:05.976Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 226577 -> 229844
7612026-07-10T05:33:06.987Zuploaded: /tmp/debug/iostat.txt
7622026-07-10T05:33:06.987Zuploading: /tmp/debug/mpstat.txt (553401 bytes)
7632026-07-10T05:33:07.002Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 553401 -> 562113
7642026-07-10T05:33:08.010Zuploaded: /tmp/debug/mpstat.txt
7652026-07-10T05:33:08.010Zuploading: /tmp/debug/paging.txt (97510 bytes)
7662026-07-10T05:33:08.021Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 97510 -> 99124
7672026-07-10T05:33:09.029Zuploaded: /tmp/debug/paging.txt
7682026-07-10T05:33:09.029Zuploading: /tmp/debug/prstat.txt (1063184 bytes)
7692026-07-10T05:33:09.048Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1063184 -> 1081307
7702026-07-10T05:33:10.058Zuploaded: /tmp/debug/prstat.txt
7712026-07-10T05:33:10.059Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7722026-07-10T05:33:11.070Zuploaded: /tmp/debug/psrinfo.txt
7732026-07-10T05:33:11.070Zuploading: /tmp/debug/upstairs-info.txt (163863 bytes)
7742026-07-10T05:33:12.087Zuploaded: /tmp/debug/upstairs-info.txt