01M5JXPP60RR81E1MYZS8WGE2G: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01M5JXQG5EZ9F6KDTPQP5HS0NW

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-10-22T23:18:32.341Zjob dependencies complete; ready to run (waiting for 14 m 9 s)
22026-10-22T23:19:25.981Zjob assigned to worker 01M5JYHSKF6QVMKZYFF6MNMX28 [factory aws, i-0a8d28359690c462f] (queued for 53 s)
32026-10-22T23:19:32.437Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-10-22T23:19:38.156Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-10-22T23:19:38.178Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-10-22T23:19:41.010Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-10-22T23:19:41.020Zdownloading input: /input/build/work/bins/crutest.gz
82026-10-22T23:19:43.143Zdownloaded input: /input/build/work/bins/crutest.gz
92026-10-22T23:19:43.148Zdownloading input: /input/build/work/bins/dsc.gz
102026-10-22T23:19:44.315Zdownloaded input: /input/build/work/bins/dsc.gz
112026-10-22T23:19:44.328Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-10-22T23:19:44.358Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-10-22T23:19:44.370Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-10-22T23:19:44.391Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-10-22T23:19:44.396Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-10-22T23:19:44.435Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-10-22T23:19:44.435Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-10-22T23:19:44.464Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-10-22T23:19:44.464Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-10-22T23:19:44.504Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-10-22T23:19:44.512Zdownloading input: /input/build/work/scripts/test_up.sh
222026-10-22T23:19:44.532Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-10-22T23:19:44.532Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-10-22T23:19:44.566Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-10-22T23:19:44.574Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-10-22T23:19:45.508Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-10-22T23:19:45.508Zstarting task 0: "setup"
282026-10-22T23:19:45.518Z++ uname -s
292026-10-22T23:19:45.518Z+ kern=SunOS
302026-10-22T23:19:45.518Z+ build_user=build
312026-10-22T23:19:45.518Z+ build_uid=12345
322026-10-22T23:19:45.518Z+ work_dir=/work
332026-10-22T23:19:45.518Z+ input_dir=/input
342026-10-22T23:19:45.518Z+ [[ 0 == 12345 ]]
352026-10-22T23:19:45.518Z+ case "$kern" in
362026-10-22T23:19:45.518Z+ groupadd -g 12345 build
372026-10-22T23:19:45.518Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-10-22T23:19:47.519Z+ zfs create -o mountpoint=/work rpool/work
392026-10-22T23:19:47.799Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-10-22T23:19:47.805Z+ home_fs=zfs
412026-10-22T23:19:47.805Z+ [[ zfs == autofs ]]
422026-10-22T23:19:47.805Z+ mkdir -p /home/build
432026-10-22T23:19:47.805Z+ chown build:build /home/build /work
442026-10-22T23:19:49.520Z+ chmod 0700 /home/build /work
452026-10-22T23:19:49.524Zprocess exited: duration 4017 ms, exit code 0
 
462026-10-22T23:19:49.533Zstarting task 1: "authentication"
472026-10-22T23:19:49.552Zprocess exited: duration 18 ms, exit code 0
 
482026-10-22T23:19:49.561Zstarting task 2: "build"
492026-10-22T23:19:49.565Z+ banner cores
502026-10-22T23:19:49.570Z
512026-10-22T23:19:49.570Z #### #### ##### ###### ####
522026-10-22T23:19:49.570Z # # # # # # # #
532026-10-22T23:19:49.570Z # # # # # ##### ####
542026-10-22T23:19:49.570Z # # # ##### # #
552026-10-22T23:19:49.570Z # # # # # # # # #
562026-10-22T23:19:49.570Z #### #### # # ###### ####
572026-10-22T23:19:49.570Z
582026-10-22T23:19:49.570Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-10-22T23:19:49.575Z+ banner unpack
602026-10-22T23:19:49.575Z
612026-10-22T23:19:49.575Z # # # # ##### ## #### # #
622026-10-22T23:19:49.575Z # # ## # # # # # # # # #
632026-10-22T23:19:49.575Z # # # # # # # # # # ####
642026-10-22T23:19:49.575Z # # # # # ##### ###### # # #
652026-10-22T23:19:49.575Z # # # ## # # # # # # #
662026-10-22T23:19:49.575Z #### # # # # # #### # #
672026-10-22T23:19:49.575Z
682026-10-22T23:19:49.575Z+ mkdir -p /var/tmp/bins
692026-10-22T23:19:49.575Z+ for t in "$input/bins/"*.gz
702026-10-22T23:19:49.580Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-10-22T23:19:49.580Z+ b=crucible-downstairs.gz
722026-10-22T23:19:49.580Z+ b=crucible-downstairs
732026-10-22T23:19:49.580Z+ gunzip
742026-10-22T23:19:52.556Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-10-22T23:19:52.560Z+ for t in "$input/bins/"*.gz
762026-10-22T23:19:52.560Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-10-22T23:19:52.560Z+ b=crucible-hammer.gz
782026-10-22T23:19:52.561Z+ b=crucible-hammer
792026-10-22T23:19:52.561Z+ gunzip
802026-10-22T23:19:55.362Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-10-22T23:19:55.367Z+ for t in "$input/bins/"*.gz
822026-10-22T23:19:55.367Z++ basename /input/build/work/bins/crutest.gz
832026-10-22T23:19:55.367Z+ b=crutest.gz
842026-10-22T23:19:55.367Z+ b=crutest
852026-10-22T23:19:55.367Z+ gunzip
862026-10-22T23:19:58.293Z+ chmod +x /var/tmp/bins/crutest
872026-10-22T23:19:58.297Z+ for t in "$input/bins/"*.gz
882026-10-22T23:19:58.297Z++ basename /input/build/work/bins/dsc.gz
892026-10-22T23:19:58.297Z+ b=dsc.gz
902026-10-22T23:19:58.297Z+ b=dsc
912026-10-22T23:19:58.297Z+ gunzip
922026-10-22T23:19:59.572Z+ chmod +x /var/tmp/bins/dsc
932026-10-22T23:19:59.577Z+ export BINDIR=/var/tmp/bins
942026-10-22T23:19:59.577Z+ BINDIR=/var/tmp/bins
952026-10-22T23:19:59.577Z+ jobpid=1076
962026-10-22T23:19:59.577Z+ echo 'Setup debug logging'
972026-10-22T23:19:59.577Z+ mkdir /tmp/debug
982026-10-22T23:19:59.577ZSetup debug logging
992026-10-22T23:19:59.577Z+ sleep 3600
1002026-10-22T23:19:59.577Z+ psrinfo -v
1012026-10-22T23:19:59.581Z+ df -h
1022026-10-22T23:19:59.585Z+ prstat -d d -mLc 1
1032026-10-22T23:19:59.585Z+ iostat -T d -xn 1
1042026-10-22T23:19:59.588Z+ mpstat -T d 1
1052026-10-22T23:19:59.588Z+ vmstat -T d -p 1
1062026-10-22T23:19:59.588Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-10-22T23:19:59.588Z+ banner test_up_2r_encrypted
1082026-10-22T23:19:59.588Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-10-22T23:19:59.588Z #####
1102026-10-22T23:19:59.589Z ##### ###### #### ##### # # ##### # # #####
1112026-10-22T23:19:59.589Z # # # # # # # # # # #
1122026-10-22T23:19:59.589Z # ##### #### # # # # # ##### # #
1132026-10-22T23:19:59.589Z # # # # # # ##### # #####
1142026-10-22T23:19:59.589Z # # # # # # # # # # #
1152026-10-22T23:19:59.589Z # ###### #### # ####### #### # ####### ####### # #
1162026-10-22T23:19:59.589Z
1172026-10-22T23:19:59.589Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-10-22T23:19:59.596Z/input/build/work
1192026-10-22T23:19:59.603ZUsing 2 region sets
1202026-10-22T23:19:59.603ZTurn off color for downstairs dump
1212026-10-22T23:19:59.614ZUpstairs using key: /Zu9mvbUu6Wt5JE343mQdWJAZfj8pBpJ4aUfmO7AUzE=
1222026-10-22T23:19:59.618Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-10-22T23:19:59.618ZCreating 6 downstairs regions
1242026-10-22T23:19:59.884ZStarting 6 downstairs
1252026-10-22T23:19:59.888Zdsc started at PID: 1139
1262026-10-22T23:20:04.976ZDisable automatic restart on all downstairs
1272026-10-22T23:20:04.989Z
1282026-10-22T23:20:04.989ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-10-22T23:20:04.992ZRunning test: span
1302026-10-22T23:20:09.078ZCompleted test: span
1312026-10-22T23:20:14.084ZRunning test: big
1322026-10-22T23:20:18.382ZCompleted test: big
1332026-10-22T23:20:23.385ZRunning test: dep
1342026-10-22T23:20:42.139ZCompleted test: dep
1352026-10-22T23:20:47.145ZRunning test: balloon
1362026-10-22T23:23:02.867ZCompleted test: balloon
1372026-10-22T23:23:07.894ZRunning test: deactivate
1382026-10-22T23:23:12.184ZCompleted test: deactivate
1392026-10-22T23:23:17.193ZRunning hammer
1402026-10-22T23:23:23.765ZRun repair tests
1412026-10-22T23:23:23.773Z/var/tmp/bins/crutest fill -g 31 -q --key /Zu9mvbUu6Wt5JE343mQdWJAZfj8pBpJ4aUfmO7AUzE= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-10-22T23:23:23.788Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.195422796Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
1432026-10-22T23:23:23.834Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.195696242Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
1442026-10-22T23:23:23.834Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.196435625Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
1452026-10-22T23:23:23.834Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.196673218Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
1462026-10-22T23:23:23.834Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"395acdb41ea0639839a46981990d46d65811ea1d\",\n git_commit_timestamp: \"2026-10-22T23:01:01.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.196708242Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
1472026-10-22T23:23:23.834Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.196722Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
1482026-10-22T23:23:23.834Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.196735158Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
1492026-10-22T23:23:23.835Z{"msg":"Crucible b04664ce-db99-4fa8-9bd4-a26b5a970b66 has session id: 1c387038-f7a0-4e95-82fc-3c482d525da0","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.197115288Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1502026-10-22T23:23:23.835Z{"msg":"Upstairs opts: Upstairs UUID: b04664ce-db99-4fa8-9bd4-a26b5a970b66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.197142123Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1512026-10-22T23:23:23.835Z{"msg":"Crucible stats registered with UUID: b04664ce-db99-4fa8-9bd4-a26b5a970b66","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.197153351Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1522026-10-22T23:23:23.835Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:22.19716377Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1532026-10-22T23:23:23.835Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.197306207Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1542026-10-22T23:23:23.835Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.197322695Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1552026-10-22T23:23:23.835Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.197334143Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1562026-10-22T23:23:23.835Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198200785Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
1572026-10-22T23:23:23.835Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198270364Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
1582026-10-22T23:23:23.835Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"395acdb41ea0639839a46981990d46d65811ea1d\",\n git_commit_timestamp: \"2026-10-22T23:01:01.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198291011Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
1592026-10-22T23:23:23.835Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.1983001Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
1602026-10-22T23:23:23.835Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198313048Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
1612026-10-22T23:23:23.835Z{"msg":"Crucible b04664ce-db99-4fa8-9bd4-a26b5a970b66 has session id: 0f314aef-cc61-4e40-80e2-b2f9d19e5de6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198521685Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
1622026-10-22T23:23:23.835Z{"msg":"Upstairs opts: Upstairs UUID: b04664ce-db99-4fa8-9bd4-a26b5a970b66, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.19855178Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
1632026-10-22T23:23:23.835Z{"msg":"Crucible stats registered with UUID: b04664ce-db99-4fa8-9bd4-a26b5a970b66","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198564178Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
1642026-10-22T23:23:23.835Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:22.198574276Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
1652026-10-22T23:23:23.835Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198588454Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
1662026-10-22T23:23:23.835Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198599982Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
1672026-10-22T23:23:23.835Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.1986115Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
1682026-10-22T23:23:23.835Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198733531Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
1692026-10-22T23:23:23.835Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198761707Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
1702026-10-22T23:23:23.835Z{"msg":"b04664ce-db99-4fa8-9bd4-a26b5a970b66 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198782623Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1712026-10-22T23:23:23.835Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198792752Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"0","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1722026-10-22T23:23:23.835Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.19880326Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"1","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1732026-10-22T23:23:23.836Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198814568Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"2","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1742026-10-22T23:23:23.836Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198829136Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1752026-10-22T23:23:23.836Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198839514Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1762026-10-22T23:23:23.836Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198850602Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1772026-10-22T23:23:23.836Z{"msg":"ds_connection connected from Ok(127.0.0.1:57062)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198898245Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1782026-10-22T23:23:23.836Z{"msg":"ds_connection connected from Ok(127.0.0.1:60554)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.198913902Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1792026-10-22T23:23:23.836Z{"msg":"ds_connection connected from Ok(127.0.0.1:41830)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.19892819Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1802026-10-22T23:23:23.836Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199583506Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"1","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1812026-10-22T23:23:23.836Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199612792Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"2","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1822026-10-22T23:23:23.836Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.19962502Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"0","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1832026-10-22T23:23:23.836Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199764198Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1842026-10-22T23:23:23.836Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199783905Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1852026-10-22T23:23:23.836Z{"msg":"[0]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199833807Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1862026-10-22T23:23:23.836Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199848874Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1872026-10-22T23:23:23.836Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199859573Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1882026-10-22T23:23:23.836Z{"msg":"[1]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199869961Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1892026-10-22T23:23:23.836Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199880549Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1902026-10-22T23:23:23.836Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199894957Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1912026-10-22T23:23:23.836Z{"msg":"[2]R flush_numbers: [4002, 4002, 4002, 4002, 4002]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199905405Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1922026-10-22T23:23:23.836Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199916044Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1932026-10-22T23:23:23.836Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199930801Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1942026-10-22T23:23:23.836Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199944629Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1952026-10-22T23:23:23.836Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199955247Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1962026-10-22T23:23:23.836Z{"msg":"Next flush: 4003","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199969235Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1972026-10-22T23:23:23.837Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199979324Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1982026-10-22T23:23:23.837Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.199993851Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
1992026-10-22T23:23:23.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.20000349Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"0","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
2002026-10-22T23:23:23.855Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200018067Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"1","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
2012026-10-22T23:23:23.855Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200028566Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"2","":"downstairs","session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
2022026-10-22T23:23:23.855Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200042993Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
2032026-10-22T23:23:23.855Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200052922Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
2042026-10-22T23:23:23.855Z{"msg":"b04664ce-db99-4fa8-9bd4-a26b5a970b66 is now active with session: 1c387038-f7a0-4e95-82fc-3c482d525da0","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.20006788Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
2052026-10-22T23:23:23.855Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200080987Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"1c387038-f7a0-4e95-82fc-3c482d525da0"}
2062026-10-22T23:23:23.855Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200090816Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
2072026-10-22T23:23:23.855Z{"msg":"Activated sub_volume b04664ce-db99-4fa8-9bd4-a26b5a970b66","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200103534Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
2082026-10-22T23:23:23.855Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200111923Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
2092026-10-22T23:23:23.856Z{"msg":"b04664ce-db99-4fa8-9bd4-a26b5a970b66 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200124291Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2102026-10-22T23:23:23.856Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200133609Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"0","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2112026-10-22T23:23:23.856Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200147767Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"1","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2122026-10-22T23:23:23.856Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200157685Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"2","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2132026-10-22T23:23:23.856Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200168074Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2142026-10-22T23:23:23.856Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200182821Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2152026-10-22T23:23:23.856Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.20019325Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2162026-10-22T23:23:23.856Z{"msg":"ds_connection connected from Ok(127.0.0.1:54883)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200207807Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"2","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2172026-10-22T23:23:23.856Z{"msg":"ds_connection connected from Ok(127.0.0.1:48100)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.20025326Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"1","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2182026-10-22T23:23:23.856Z{"msg":"ds_connection connected from Ok(127.0.0.1:54014)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200295503Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"io task","client":"0","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2192026-10-22T23:23:23.856Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200578289Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"1","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2202026-10-22T23:23:23.856Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200598995Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"0","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2212026-10-22T23:23:23.856Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200655986Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"2","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2222026-10-22T23:23:23.856Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200745262Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2232026-10-22T23:23:23.856Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200782396Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2242026-10-22T23:23:23.856Z{"msg":"[0]R flush_numbers: [5323, 5323, 5323, 5323, 5320]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200793305Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2252026-10-22T23:23:23.856Z{"msg":"[0]R generation: [14, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200803833Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2262026-10-22T23:23:23.856Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200814771Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2272026-10-22T23:23:23.856Z{"msg":"[1]R flush_numbers: [5323, 5323, 5323, 5323, 5320]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200825229Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2282026-10-22T23:23:23.856Z{"msg":"[1]R generation: [14, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200836048Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2292026-10-22T23:23:23.856Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200847146Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2302026-10-22T23:23:23.856Z{"msg":"[2]R flush_numbers: [5323, 5323, 5323, 5323, 5320]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200858084Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2312026-10-22T23:23:23.856Z{"msg":"[2]R generation: [14, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200868693Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2322026-10-22T23:23:23.856Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200879711Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2332026-10-22T23:23:23.857Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200890329Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2342026-10-22T23:23:23.857Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200901217Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2352026-10-22T23:23:23.857Z{"msg":"Next flush: 5324","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200911526Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2362026-10-22T23:23:23.857Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200921804Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"":"downstairs","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2372026-10-22T23:23:23.857Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200932402Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2382026-10-22T23:23:23.857Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200942231Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"0","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2392026-10-22T23:23:23.857Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200952699Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"1","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2402026-10-22T23:23:23.857Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200963208Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"client":"2","":"downstairs","session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2412026-10-22T23:23:23.857Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200973836Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2422026-10-22T23:23:23.859Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200984054Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2432026-10-22T23:23:23.886Z{"msg":"b04664ce-db99-4fa8-9bd4-a26b5a970b66 is now active with session: 0f314aef-cc61-4e40-80e2-b2f9d19e5de6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.200995063Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2442026-10-22T23:23:23.886Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.201004751Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"session_id":"0f314aef-cc61-4e40-80e2-b2f9d19e5de6"}
2452026-10-22T23:23:23.887Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.201015169Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
2462026-10-22T23:23:23.887Z{"msg":"Activated sub_volume b04664ce-db99-4fa8-9bd4-a26b5a970b66","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.201024338Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
2472026-10-22T23:23:23.887Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.201033506Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
2482026-10-22T23:23:23.887ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2492026-10-22T23:23:23.887Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.201082669Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
2502026-10-22T23:23:23.887ZRead and Verify all blocks (0..100 range:false)
2512026-10-22T23:23:23.887Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.268194661Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
2522026-10-22T23:23:23.887Z----------------------------------------------------------------
2532026-10-22T23:23:23.887Z{ Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2542026-10-22T23:23:23.887Z"States:msg":" Active Active ActiveCLIENT: Tests done. All submitted work has been ACK'd
2552026-10-22T23:23:23.887Z","v":0,"name":"crucible","level":30JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2562026-10-22T23:23:23.887Z,"time":"2026-10-22T23:23:22.268228696Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest" 1002 Acked Read 50}
2572026-10-22T23:23:23.887Z Done Done Done false
2582026-10-22T23:23:23.887Z STATES DS:0 DS:1 DS:2 TOTAL
2592026-10-22T23:23:23.887Z Sent 0 0 0 0
2602026-10-22T23:23:23.887Z Done 1 1 1 3
2612026-10-22T23:23:23.887Z Skipped 0 0 0 0
2622026-10-22T23:23:23.887Z Error 0 0 0 0
2632026-10-22T23:23:23.887ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2642026-10-22T23:23:23.887ZDownstairs last five completed: 1001 1000
2652026-10-22T23:23:23.887ZUpstairs last five completed: 1002 1001 1000
2662026-10-22T23:23:23.887Z----------------------------------------------------------------
2672026-10-22T23:23:23.887Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2682026-10-22T23:23:23.888ZStates: Active Active Active
2692026-10-22T23:23:23.888ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2702026-10-22T23:23:23.888Z 1002 Acked Read 50 Done Done Done false
2712026-10-22T23:23:23.888Z STATES DS:0 DS:1 DS:2 TOTAL
2722026-10-22T23:23:23.888Z Sent 0 0 0 0
2732026-10-22T23:23:23.888Z Done 1 1 1 3
2742026-10-22T23:23:23.888Z Skipped 0 0 0 0
2752026-10-22T23:23:23.888Z Error 0 0 0 0
2762026-10-22T23:23:23.888ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2772026-10-22T23:23:23.888ZDownstairs last five completed: 1001 1000
2782026-10-22T23:23:23.888ZUpstairs last five completed: 1002 1001 1000
2792026-10-22T23:23:23.888Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:22.268334899Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
2802026-10-22T23:23:27.862Z----------------------------------------------------------------
2812026-10-22T23:23:27.862Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2822026-10-22T23:23:27.862ZDownstairs last five completed: 1003 1002 1001 1000
2832026-10-22T23:23:27.869ZUpstairs last five completed: 1003 1002 1001 1000
2842026-10-22T23:23:27.881Z----------------------------------------------------------------
2852026-10-22T23:23:27.881Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2862026-10-22T23:23:27.881ZDownstairs last five completed: 1003 1002 1001 1000
2872026-10-22T23:23:27.881ZUpstairs last five completed: 1003 1002 1001 1000
2882026-10-22T23:23:27.881Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.269582181Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
2892026-10-22T23:23:27.881Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.269635853Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182,"task":"crutest"}
2902026-10-22T23:23:27.881Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:26.269647521Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1182}
2912026-10-22T23:23:27.881ZRepair setup passed
2922026-10-22T23:23:27.881ZCopy the region for /var/tmp/test_up-build/8830
2932026-10-22T23:23:27.881Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2942026-10-22T23:23:27.881Z/var/tmp/bins/crutest fill -g 32 -q --key /Zu9mvbUu6Wt5JE343mQdWJAZfj8pBpJ4aUfmO7AUzE= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2952026-10-22T23:23:27.894Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.301421844Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
2962026-10-22T23:23:27.899Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.301809413Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
2972026-10-22T23:23:27.899Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.302598558Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
2982026-10-22T23:23:27.899Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.302821732Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
2992026-10-22T23:23:27.899Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"395acdb41ea0639839a46981990d46d65811ea1d\",\n git_commit_timestamp: \"2026-10-22T23:01:01.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.302852008Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3002026-10-22T23:23:27.900Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.302865555Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3012026-10-22T23:23:27.900Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.302878263Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3022026-10-22T23:23:27.900Z{"msg":"Crucible 2470fb10-be44-4f51-b656-94a9d731be2d has session id: 3901db91-3090-4724-aa73-ee48df332cbc","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.303311235Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3032026-10-22T23:23:27.900Z{"msg":"Upstairs opts: Upstairs UUID: 2470fb10-be44-4f51-b656-94a9d731be2d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.303352648Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3042026-10-22T23:23:27.900Z{"msg":"Crucible stats registered with UUID: 2470fb10-be44-4f51-b656-94a9d731be2d","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.303368786Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3052026-10-22T23:23:27.900Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:26.303378914Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3062026-10-22T23:23:27.900Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.303489327Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3072026-10-22T23:23:27.900Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.303510563Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3082026-10-22T23:23:27.903Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.303521321Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3092026-10-22T23:23:27.903Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.30441512Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
3102026-10-22T23:23:27.903Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304462542Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3112026-10-22T23:23:27.903Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"395acdb41ea0639839a46981990d46d65811ea1d\",\n git_commit_timestamp: \"2026-10-22T23:01:01.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304488868Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3122026-10-22T23:23:27.903Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304502226Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3132026-10-22T23:23:27.903Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304514254Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3142026-10-22T23:23:27.904Z{"msg":"Crucible 2470fb10-be44-4f51-b656-94a9d731be2d has session id: ffa51428-62bf-45a3-add8-4d6699266d83","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304752476Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3152026-10-22T23:23:27.904Z{"msg":"Upstairs opts: Upstairs UUID: 2470fb10-be44-4f51-b656-94a9d731be2d, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304776323Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3162026-10-22T23:23:27.904Z{"msg":"Crucible stats registered with UUID: 2470fb10-be44-4f51-b656-94a9d731be2d","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304787031Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3172026-10-22T23:23:27.904Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:26.304803898Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3182026-10-22T23:23:27.904Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304815806Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3192026-10-22T23:23:27.904Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304826865Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3202026-10-22T23:23:27.904Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304841562Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3212026-10-22T23:23:27.904Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304962863Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
3222026-10-22T23:23:27.904Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.304976571Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3232026-10-22T23:23:27.904Z{"msg":"2470fb10-be44-4f51-b656-94a9d731be2d active request set","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305013855Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3242026-10-22T23:23:27.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305029922Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"0","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3252026-10-22T23:23:27.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305040621Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"1","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3262026-10-22T23:23:27.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305068516Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"2","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3272026-10-22T23:23:27.904Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305078615Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3282026-10-22T23:23:27.904Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305089163Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3292026-10-22T23:23:27.904Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305099671Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3302026-10-22T23:23:27.904Z{"msg":"ds_connection connected from Ok(127.0.0.1:57797)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.30511044Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3312026-10-22T23:23:27.904Z{"msg":"ds_connection connected from Ok(127.0.0.1:61401)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305141165Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3322026-10-22T23:23:27.904Z{"msg":"ds_connection connected from Ok(127.0.0.1:50334)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305214503Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3332026-10-22T23:23:27.904Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305850162Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"2","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3342026-10-22T23:23:27.913Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305950746Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"0","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3352026-10-22T23:23:27.913Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.305971703Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"1","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3362026-10-22T23:23:27.913Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306103652Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3372026-10-22T23:23:27.913Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306136657Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3382026-10-22T23:23:27.913Z{"msg":"[0]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.30617925Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3392026-10-22T23:23:27.913Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306190998Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3402026-10-22T23:23:27.913Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306202127Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3412026-10-22T23:23:27.913Z{"msg":"[1]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306213195Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3422026-10-22T23:23:27.913Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306223773Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3432026-10-22T23:23:27.913Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306234691Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3442026-10-22T23:23:27.913Z{"msg":"[2]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.30624578Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3452026-10-22T23:23:27.913Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306263567Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3462026-10-22T23:23:27.913Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306273905Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3472026-10-22T23:23:27.913Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306284254Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3482026-10-22T23:23:27.913Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306294512Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3492026-10-22T23:23:27.913Z{"msg":"Next flush: 4004","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.30630474Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3502026-10-22T23:23:27.913Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306314859Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3512026-10-22T23:23:27.913Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306332876Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3522026-10-22T23:23:27.913Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306342234Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"0","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3532026-10-22T23:23:27.913Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306352873Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"1","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3542026-10-22T23:23:27.913Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306363331Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"2","":"downstairs","session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3552026-10-22T23:23:27.914Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306374109Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3562026-10-22T23:23:27.914Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306384238Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3572026-10-22T23:23:27.914Z{"msg":"2470fb10-be44-4f51-b656-94a9d731be2d is now active with session: 3901db91-3090-4724-aa73-ee48df332cbc","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306395656Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3582026-10-22T23:23:27.914Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306405274Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"3901db91-3090-4724-aa73-ee48df332cbc"}
3592026-10-22T23:23:27.914Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306415543Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3602026-10-22T23:23:27.914Z{"msg":"Activated sub_volume 2470fb10-be44-4f51-b656-94a9d731be2d","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306424501Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3612026-10-22T23:23:27.914Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.30643376Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3622026-10-22T23:23:27.914Z{"msg":"2470fb10-be44-4f51-b656-94a9d731be2d active request set","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306442109Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3632026-10-22T23:23:27.914Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306451537Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"0","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3642026-10-22T23:23:27.914Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306461955Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"1","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3652026-10-22T23:23:27.914Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306479473Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"2","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3662026-10-22T23:23:27.914Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306490031Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3672026-10-22T23:23:27.914Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306500639Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3682026-10-22T23:23:27.914Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306510908Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3692026-10-22T23:23:27.914Z{"msg":"ds_connection connected from Ok(127.0.0.1:53848)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.3066194Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3702026-10-22T23:23:27.918Z{"msg":"ds_connection connected from Ok(127.0.0.1:49103)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306656884Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3712026-10-22T23:23:27.918Z{"msg":"ds_connection connected from Ok(127.0.0.1:42018)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306669772Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3722026-10-22T23:23:27.918Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.306980873Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"2","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3732026-10-22T23:23:27.918Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.30700095Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"0","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3742026-10-22T23:23:27.918Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307013978Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"1","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3752026-10-22T23:23:27.918Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307144777Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3762026-10-22T23:23:27.918Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307204238Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3772026-10-22T23:23:27.918Z{"msg":"[0]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307271707Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3782026-10-22T23:23:27.918Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307282995Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3792026-10-22T23:23:27.918Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307297633Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3802026-10-22T23:23:27.918Z{"msg":"[1]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307308211Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3812026-10-22T23:23:27.918Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307322789Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3822026-10-22T23:23:27.918Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307333417Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3832026-10-22T23:23:27.918Z{"msg":"[2]R flush_numbers: [5324, 5324, 5324, 5324, 5324]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307347845Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3842026-10-22T23:23:27.918Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307361903Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3852026-10-22T23:23:27.918Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307373131Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3862026-10-22T23:23:27.918Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307387159Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3872026-10-22T23:23:27.918Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307397077Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3882026-10-22T23:23:27.918Z{"msg":"Next flush: 5325","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307407256Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3892026-10-22T23:23:27.918Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307421233Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"":"downstairs","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3902026-10-22T23:23:27.918Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307431422Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3912026-10-22T23:23:27.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.30744493Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"0","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3922026-10-22T23:23:27.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307455478Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"1","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3932026-10-22T23:23:27.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307469496Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"client":"2","":"downstairs","session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3942026-10-22T23:23:27.919Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307480204Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3952026-10-22T23:23:27.919Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307493892Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3962026-10-22T23:23:27.919Z{"msg":"2470fb10-be44-4f51-b656-94a9d731be2d is now active with session: ffa51428-62bf-45a3-add8-4d6699266d83","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.30750491Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3972026-10-22T23:23:27.919Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307518318Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"session_id":"ffa51428-62bf-45a3-add8-4d6699266d83"}
3982026-10-22T23:23:27.919Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307527996Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
3992026-10-22T23:23:27.919Z{"msg":"Activated sub_volume 2470fb10-be44-4f51-b656-94a9d731be2d","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307540324Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
4002026-10-22T23:23:27.919Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307548683Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
4012026-10-22T23:23:27.919ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4022026-10-22T23:23:27.919ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4032026-10-22T23:23:27.919Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.307871522Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
4042026-10-22T23:23:27.932ZRead and Verify all blocks (0..100 range:false)
4052026-10-22T23:23:27.975Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.374995243Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
4062026-10-22T23:23:27.975Z----------------------------------------------------------------
4072026-10-22T23:23:27.982Z{ Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4082026-10-22T23:23:27.982Z"States:msg":" Active Active Active
4092026-10-22T23:23:27.982ZCLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4102026-10-22T23:23:27.982Z,"time":"2026-10-22T23:23:26.375028777Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest" 1002 Acked Read 50}
4112026-10-22T23:23:27.982Z Done Done Done false
4122026-10-22T23:23:27.982Z STATES DS:0 DS:1 DS:2 TOTAL
4132026-10-22T23:23:27.983Z Sent 0 0 0 0
4142026-10-22T23:23:27.983Z Done 1 1 1 3
4152026-10-22T23:23:27.983Z Skipped 0 0 0 0
4162026-10-22T23:23:27.983Z Error 0 0 0 0
4172026-10-22T23:23:27.983ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4182026-10-22T23:23:27.983ZDownstairs last five completed: 1001 1000
4192026-10-22T23:23:27.983ZUpstairs last five completed: 1002 1001 1000
4202026-10-22T23:23:28.022Z----------------------------------------------------------------
4212026-10-22T23:23:28.022Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4222026-10-22T23:23:28.022ZStates: Active Active Active
4232026-10-22T23:23:28.022ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4242026-10-22T23:23:28.022Z 1002 Acked Read 50 Done Done Done false
4252026-10-22T23:23:28.022Z STATES DS:0 DS:1 DS:2 TOTAL
4262026-10-22T23:23:28.022Z Sent 0 0 0 0
4272026-10-22T23:23:28.022Z Done 1 1 1 3
4282026-10-22T23:23:28.022Z Skipped 0 0 0 0
4292026-10-22T23:23:28.022Z Error 0 0 0 0
4302026-10-22T23:23:28.022ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4312026-10-22T23:23:28.022ZDownstairs last five completed: 1001 1000
4322026-10-22T23:23:28.022ZUpstairs last five completed: 1002 1001 1000
4332026-10-22T23:23:28.022Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:26.375142949Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
4342026-10-22T23:23:31.968Z----------------------------------------------------------------
4352026-10-22T23:23:31.968Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4362026-10-22T23:23:31.968ZDownstairs last five completed: 1003 1002 1001 1000
4372026-10-22T23:23:31.969ZUpstairs last five completed: 1003 1002 1001 1000
4382026-10-22T23:23:31.972Z----------------------------------------------------------------
4392026-10-22T23:23:31.972Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4402026-10-22T23:23:31.973ZDownstairs last five completed: 1003 1002 1001 1000
4412026-10-22T23:23:31.973ZUpstairs last five completed: 1003 1002 1001 1000
4422026-10-22T23:23:31.973Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.375511281Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
4432026-10-22T23:23:31.973Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.375575571Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188,"task":"crutest"}
4442026-10-22T23:23:31.973Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:30.375587169Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1188}
4452026-10-22T23:23:31.976ZRepair part 1 passed
4462026-10-22T23:23:31.976Z
4472026-10-22T23:23:31.976ZKill the current downstairs
4482026-10-22T23:23:32.017ZDownstairs 2 stopped
4492026-10-22T23:23:32.017Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4502026-10-22T23:23:32.021ZNow put back the original so we have a mismatch
4512026-10-22T23:23:32.021Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4522026-10-22T23:23:32.021ZRestart downstairs with old directory
4532026-10-22T23:23:32.033Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4542026-10-22T23:23:32.045ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4552026-10-22T23:23:32.049Z 0 000-009 32 32 31 4004 4004 4003 F F F <---
4562026-10-22T23:23:32.049Z 1 010-019 32 32 31 4004 4004 4003 F F F <---
4572026-10-22T23:23:32.049Z 2 020-029 32 32 31 4004 4004 4003 F F F <---
4582026-10-22T23:23:32.049Z 3 030-039 32 32 31 4004 4004 4003 F F F <---
4592026-10-22T23:23:32.049Z 4 040-049 32 32 31 4004 4004 4003 F F F <---
4602026-10-22T23:23:32.049ZMax gen: 32, Max flush: 4004
4612026-10-22T23:23:32.049ZError: Difference in extent metadata found!
4622026-10-22T23:23:32.049Zdump test found error as expected
4632026-10-22T23:23:32.049Z
4642026-10-22T23:23:32.049Z
4652026-10-22T23:23:32.049Z/var/tmp/bins/crutest verify --range -g 33 -q --key /Zu9mvbUu6Wt5JE343mQdWJAZfj8pBpJ4aUfmO7AUzE= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4662026-10-22T23:23:32.066Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.473297601Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
4672026-10-22T23:23:32.070Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.473633188Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
4682026-10-22T23:23:32.070Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.474307921Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
4692026-10-22T23:23:32.070Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.474543594Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
4702026-10-22T23:23:32.070Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"395acdb41ea0639839a46981990d46d65811ea1d\",\n git_commit_timestamp: \"2026-10-22T23:01:01.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.474584237Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
4712026-10-22T23:23:32.070Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.474597885Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
4722026-10-22T23:23:32.070Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.474610333Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
4732026-10-22T23:23:32.070Z{"msg":"Crucible 7522ea64-f9a4-402c-baf4-9231616b9818 has session id: dcf8ced1-00c2-4ac9-8075-ee869478e845","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.474978225Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4742026-10-22T23:23:32.070Z{"msg":"Upstairs opts: Upstairs UUID: 7522ea64-f9a4-402c-baf4-9231616b9818, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.475002051Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4752026-10-22T23:23:32.070Z{"msg":"Crucible stats registered with UUID: 7522ea64-f9a4-402c-baf4-9231616b9818","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.475012529Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4762026-10-22T23:23:32.070Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:30.475022718Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4772026-10-22T23:23:32.071Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.475182162Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4782026-10-22T23:23:32.071Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.47519864Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4792026-10-22T23:23:32.071Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.475209928Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4802026-10-22T23:23:32.071Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476068892Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
4812026-10-22T23:23:32.071Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476136511Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
4822026-10-22T23:23:32.071Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"395acdb41ea0639839a46981990d46d65811ea1d\",\n git_commit_timestamp: \"2026-10-22T23:01:01.000000000Z\",\n git_branch: \"renovate/crate-rand-vulnerability\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476157098Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
4832026-10-22T23:23:32.071Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476166166Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
4842026-10-22T23:23:32.071Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476178295Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
4852026-10-22T23:23:32.071Z{"msg":"Crucible 7522ea64-f9a4-402c-baf4-9231616b9818 has session id: 79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476374913Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
4862026-10-22T23:23:32.071Z{"msg":"Upstairs opts: Upstairs UUID: 7522ea64-f9a4-402c-baf4-9231616b9818, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476411098Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
4872026-10-22T23:23:32.071Z{"msg":"Crucible stats registered with UUID: 7522ea64-f9a4-402c-baf4-9231616b9818","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476428305Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
4882026-10-22T23:23:32.071Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:30.476438573Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
4892026-10-22T23:23:32.071Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476448622Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
4902026-10-22T23:23:32.071Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.47646021Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
4912026-10-22T23:23:32.071Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476471478Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
4922026-10-22T23:23:32.071Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476609996Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
4932026-10-22T23:23:32.071Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476629973Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
4942026-10-22T23:23:32.071Z{"msg":"7522ea64-f9a4-402c-baf4-9231616b9818 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476638852Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4952026-10-22T23:23:32.071Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.47664897Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"0","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4962026-10-22T23:23:32.071Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476659658Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"1","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4972026-10-22T23:23:32.072Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476670297Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4982026-10-22T23:23:32.072Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476681015Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
4992026-10-22T23:23:32.072Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476691763Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5002026-10-22T23:23:32.072Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.47671329Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5012026-10-22T23:23:32.072Z{"msg":"ds_connection connected from Ok(127.0.0.1:44260)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476723828Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5022026-10-22T23:23:32.072Z{"msg":"ds_connection connected from Ok(127.0.0.1:46624)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476827022Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5032026-10-22T23:23:32.072Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:30.476856327Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5042026-10-22T23:23:32.072Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:30.476869655Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5052026-10-22T23:23:32.072Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:30.476893281Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5062026-10-22T23:23:32.072Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.47690305Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5072026-10-22T23:23:32.072Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.476922357Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5082026-10-22T23:23:32.076Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.477428846Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"1","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5092026-10-22T23:23:32.076Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.47746709Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"0","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5102026-10-22T23:23:32.076Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.477602159Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5112026-10-22T23:23:32.076Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:30.477617216Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5122026-10-22T23:23:42.072Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.477627583Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5132026-10-22T23:23:42.072Z{"msg":"ds_connection connected from Ok(127.0.0.1:62722)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.477687683Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5142026-10-22T23:23:42.079Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478572013Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5152026-10-22T23:23:42.079Z{"msg":"[0]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478759883Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5162026-10-22T23:23:42.079Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478775711Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5172026-10-22T23:23:42.079Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478786419Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5182026-10-22T23:23:42.079Z{"msg":"[1]R flush_numbers: [4004, 4004, 4004, 4004, 4004]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478797007Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5192026-10-22T23:23:42.079Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478811755Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5202026-10-22T23:23:42.079Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478822253Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5212026-10-22T23:23:42.079Z{"msg":"[2]R flush_numbers: [4003, 4003, 4003, 4003, 4003]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478836911Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5222026-10-22T23:23:42.079Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478847049Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5232026-10-22T23:23:42.079Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478861607Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5242026-10-22T23:23:42.079Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478871476Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5252026-10-22T23:23:42.079Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478886153Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5262026-10-22T23:23:42.079Z{"msg":"Next flush: 4005","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478895842Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5272026-10-22T23:23:42.080Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4003, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478914399Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5282026-10-22T23:23:42.080Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4003, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478933576Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5292026-10-22T23:23:42.080Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478949013Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5302026-10-22T23:23:42.080Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4003, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478963231Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5312026-10-22T23:23:42.080Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4003, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478981438Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5322026-10-22T23:23:42.080Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.478996406Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5332026-10-22T23:23:42.080Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4003, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479022152Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5342026-10-22T23:23:42.080Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4003, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479041389Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5352026-10-22T23:23:42.080Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479056616Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5362026-10-22T23:23:42.080Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4003, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479073354Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5372026-10-22T23:23:42.080Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4003, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479091701Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5382026-10-22T23:23:42.080Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479102829Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5392026-10-22T23:23:42.080Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4003, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479121166Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5402026-10-22T23:23:42.080Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 4004, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 4003, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479139133Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5412026-10-22T23:23:42.082Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479154341Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"mend","":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5422026-10-22T23:23:42.082Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479173088Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"0","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5432026-10-22T23:23:42.082Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479188085Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"1","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5442026-10-22T23:23:42.082Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479198734Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5452026-10-22T23:23:42.082Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479226019Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5462026-10-22T23:23:42.082Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4005, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4005, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4005, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4005, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4005, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479319495Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5472026-10-22T23:23:42.083Z{"msg":"starting reconciliation 6a74c79a-183f-4501-aaed-961719e2a183: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479331383Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5482026-10-22T23:23:42.083Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.479341581Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5492026-10-22T23:23:42.088Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.482900497Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5502026-10-22T23:23:42.088Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.484926006Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5512026-10-22T23:23:42.088Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.487081744Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5522026-10-22T23:23:42.088Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.488979473Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5532026-10-22T23:23:42.088Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490693302Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5542026-10-22T23:23:42.089Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490717798Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"0","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5552026-10-22T23:23:42.089Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490729106Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"1","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5562026-10-22T23:23:42.089Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490740094Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5572026-10-22T23:23:42.089Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490751383Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5582026-10-22T23:23:42.089Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490761651Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5592026-10-22T23:23:42.089Z{"msg":"7522ea64-f9a4-402c-baf4-9231616b9818 is now active with session: dcf8ced1-00c2-4ac9-8075-ee869478e845","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490772699Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"dcf8ced1-00c2-4ac9-8075-ee869478e845"}
5602026-10-22T23:23:42.089Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490783208Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
5612026-10-22T23:23:42.089Z{"msg":"Activated sub_volume 7522ea64-f9a4-402c-baf4-9231616b9818","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.49083272Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
5622026-10-22T23:23:42.089Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.49089157Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
5632026-10-22T23:23:42.089Z{"msg":"7522ea64-f9a4-402c-baf4-9231616b9818 active request set","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490906318Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5642026-10-22T23:23:42.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490916066Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"0","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5652026-10-22T23:23:42.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490926655Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"1","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5662026-10-22T23:23:42.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490936473Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5672026-10-22T23:23:42.089Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490950931Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5682026-10-22T23:23:42.089Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490961729Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5692026-10-22T23:23:42.089Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490972608Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5702026-10-22T23:23:42.089Z{"msg":"ds_connection connected from Ok(127.0.0.1:53759)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490983686Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5712026-10-22T23:23:42.090Z{"msg":"ds_connection connected from Ok(127.0.0.1:36196)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.490998053Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5722026-10-22T23:23:42.091Z{"msg":"ds_connection connected from Ok(127.0.0.1:58127)","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491014601Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5732026-10-22T23:23:42.091Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491390291Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5742026-10-22T23:23:42.091Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491415017Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"1","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5752026-10-22T23:23:42.091Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491426276Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"0","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5762026-10-22T23:23:42.091Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491520281Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5772026-10-22T23:23:42.091Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491548426Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5782026-10-22T23:23:42.091Z{"msg":"[0]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491591909Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5792026-10-22T23:23:42.091Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491607647Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5802026-10-22T23:23:42.091Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491618085Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5812026-10-22T23:23:42.091Z{"msg":"[1]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491632573Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5822026-10-22T23:23:42.091Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491642551Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5832026-10-22T23:23:42.091Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491657319Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5842026-10-22T23:23:42.091Z{"msg":"[2]R flush_numbers: [5325, 5325, 5325, 5325, 5325]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491667737Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5852026-10-22T23:23:42.091Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491678146Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5862026-10-22T23:23:42.091Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491688534Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5872026-10-22T23:23:42.091Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491702432Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5882026-10-22T23:23:42.091Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.49171244Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5892026-10-22T23:23:42.091Z{"msg":"Next flush: 5326","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491722729Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5902026-10-22T23:23:42.091Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491736696Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"":"downstairs","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5912026-10-22T23:23:42.091Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491747055Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5922026-10-22T23:23:42.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491760353Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"0","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5932026-10-22T23:23:42.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491770531Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"1","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5942026-10-22T23:23:42.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491784549Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"client":"2","":"downstairs","session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5952026-10-22T23:23:42.093Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491794557Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5962026-10-22T23:23:42.093Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491808405Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5972026-10-22T23:23:42.093Z{"msg":"7522ea64-f9a4-402c-baf4-9231616b9818 is now active with session: 79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491818953Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
5982026-10-22T23:23:42.093Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491832621Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
5992026-10-22T23:23:42.093Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.49184111Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"session_id":"79de4d4c-1d55-47bc-91c5-7b5c74e1fa6d"}
6002026-10-22T23:23:42.093Z{"msg":"Activated sub_volume 7522ea64-f9a4-402c-baf4-9231616b9818","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491855058Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
6012026-10-22T23:23:42.093Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.491863806Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
6022026-10-22T23:23:42.093ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6032026-10-22T23:23:42.093ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6042026-10-22T23:23:42.093ZRead and Verify all blocks (0..100 range:true)
6052026-10-22T23:23:42.132Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.53732863Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
6062026-10-22T23:23:42.132Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.537351246Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
6072026-10-22T23:23:42.136Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.537394269Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
6082026-10-22T23:23:42.136Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.537404788Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
6092026-10-22T23:23:42.136Z----------------------------------------------------------------
6102026-10-22T23:23:42.136Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6112026-10-22T23:23:42.136ZStates: Active Active Active
6122026-10-22T23:23:42.136ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6132026-10-22T23:23:42.136Z 1000 Acked Read 50 Done Done Done false
6142026-10-22T23:23:42.136Z STATES DS:0 DS:1 DS:2 TOTAL
6152026-10-22T23:23:42.136Z Sent 0 0 0 0
6162026-10-22T23:23:42.136Z Done 1 1 1 3
6172026-10-22T23:23:42.136Z Skipped 0 0 0 0
6182026-10-22T23:23:42.136Z Error 0 0 0 0
6192026-10-22T23:23:42.136ZLast Flush: None None None
6202026-10-22T23:23:42.137ZDownstairs last five completed:
6212026-10-22T23:23:42.137ZUpstairs last five completed: 1000
6222026-10-22T23:23:42.137Z----------------------------------------------------------------
6232026-10-22T23:23:42.137Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6242026-10-22T23:23:42.137ZStates: Active Active Active
6252026-10-22T23:23:42.137ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6262026-10-22T23:23:42.137Z 1000 Acked Read 50 Done Done Done false
6272026-10-22T23:23:42.137Z STATES DS:0 DS:1 DS:2 TOTAL
6282026-10-22T23:23:42.137Z Sent 0 0 0 0
6292026-10-22T23:23:42.137Z Done 1 1 1 3
6302026-10-22T23:23:42.137Z Skipped 0 0 0 0
6312026-10-22T23:23:42.138Z Error 0 0 0 0
6322026-10-22T23:23:42.138ZLast Flush: None None None
6332026-10-22T23:23:42.138ZDownstairs last five completed:
6342026-10-22T23:23:42.138ZUpstairs last five completed: 1000
6352026-10-22T23:23:42.138Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:40.537541576Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
6362026-10-22T23:23:46.134Z----------------------------------------------------------------
6372026-10-22T23:23:46.134Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6382026-10-22T23:23:46.134ZDownstairs last five completed: 1001 1000
6392026-10-22T23:23:46.134ZUpstairs last five completed: 1001 1000
6402026-10-22T23:23:46.134Z----------------------------------------------------------------
6412026-10-22T23:23:46.134Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6422026-10-22T23:23:46.134ZDownstairs last five completed: 1001 1000
6432026-10-22T23:23:46.134ZUpstairs last five completed: 1001 1000
6442026-10-22T23:23:46.138Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:44.538747305Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
6452026-10-22T23:23:46.138Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-10-22T23:23:44.538804856Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196,"task":"crutest"}
6462026-10-22T23:23:46.138Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-10-22T23:23:44.538816634Z","hostname":"w-01M5JYHSKF6QVMKZYFF6MNMX28","pid":1196}
6472026-10-22T23:23:46.141ZRepair part 2 passed
6482026-10-22T23:23:46.141Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6492026-10-22T23:23:46.153ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6502026-10-22T23:23:46.157Z 0 000-009 32 32 32 4004 4004 4004 F F F
6512026-10-22T23:23:46.157Z 1 010-019 32 32 32 4004 4004 4004 F F F
6522026-10-22T23:23:46.157Z 2 020-029 32 32 32 4004 4004 4004 F F F
6532026-10-22T23:23:46.157Z 3 030-039 32 32 32 4004 4004 4004 F F F
6542026-10-22T23:23:46.157Z 4 040-049 32 32 32 4004 4004 4004 F F F
6552026-10-22T23:23:46.157ZMax gen: 32, Max flush: 4004
6562026-10-22T23:23:46.157Zdump test passed
6572026-10-22T23:23:46.157Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6582026-10-22T23:23:46.166Z Extent 1
6592026-10-22T23:23:46.170ZGEN 32 32 32
6602026-10-22T23:23:46.170ZFLUSH_ID 4004 4004 4004
6612026-10-22T23:23:46.170ZDIRTY
6622026-10-22T23:23:46.170Z
6632026-10-22T23:23:46.173ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6642026-10-22T23:23:46.173Z010 A A A A A A
6652026-10-22T23:23:46.173Z011 A A A A A A
6662026-10-22T23:23:46.173Z012 A A A A A A
6672026-10-22T23:23:46.173Z013 A A A A A A
6682026-10-22T23:23:46.173Z014 A A A A A A
6692026-10-22T23:23:46.173Z015 A A A A A A
6702026-10-22T23:23:46.177Z016 A A A A A A
6712026-10-22T23:23:46.177Z017 A A A A A A
6722026-10-22T23:23:46.177Z018 A A A A A A
6732026-10-22T23:23:46.177Z019 A A A A A A
6742026-10-22T23:23:46.177Zdump extent test passed
6752026-10-22T23:23:46.177Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6762026-10-22T23:23:46.187ZExtent 2 Block in extent 0 Actual block 20
6772026-10-22T23:23:46.206Z
6782026-10-22T23:23:46.206Z DATA SHA256 VER
6792026-10-22T23:23:46.206Z------ ---------------------------------------------------------------- ---
6802026-10-22T23:23:46.206Z 0 f8c2c643c88b4bf62585bb9beea22cad7053b42ed07df36db4cd423aad0a9d31 A
6812026-10-22T23:23:46.206Z 1 f8c2c643c88b4bf62585bb9beea22cad7053b42ed07df36db4cd423aad0a9d31 A
6822026-10-22T23:23:46.206Z 2 f8c2c643c88b4bf62585bb9beea22cad7053b42ed07df36db4cd423aad0a9d31 A
6832026-10-22T23:23:46.207Z
6842026-10-22T23:23:46.207ZNONCES 0 1 2 DIFF
6852026-10-22T23:23:46.207Z------ ------------------------ ------------------------ ------------------------ -----
6862026-10-22T23:23:46.207Z 0 60ba32b317364fc554921b0c 60ba32b317364fc554921b0c 60ba32b317364fc554921b0c
6872026-10-22T23:23:46.207Z
6882026-10-22T23:23:46.207Z TAGS 0 1 2 DIFF
6892026-10-22T23:23:46.207Z------ -------------------------------- -------------------------------- -------------------------------- -----
6902026-10-22T23:23:46.207Z 0 b33fb6e86776e2281378a337cfafa838 b33fb6e86776e2281378a337cfafa838 b33fb6e86776e2281378a337cfafa838
6912026-10-22T23:23:46.207Z
6922026-10-22T23:23:46.207ZHASHES 0 1 2 DIFF
6932026-10-22T23:23:46.208Z------ ---------------- ---------------- ---------------- -----
6942026-10-22T23:23:46.208Z
6952026-10-22T23:23:46.208Zdump block test passed
6962026-10-22T23:23:46.208ZInitial upstairs tests have completed, stopping all downstairs
6972026-10-22T23:23:51.209ZCreating 7 larger downstairs regions
6982026-10-22T23:23:51.414ZStarting 7 downstairs
6992026-10-22T23:23:56.420Zdsc restarted at PID: 1212
7002026-10-22T23:23:56.519ZNow do the replace-reconcile test
7012026-10-22T23:23:56.532ZUsing 8870 for the replacement port
7022026-10-22T23:27:27.515ZCompleted test: replace-reconcile
7032026-10-22T23:27:27.519ZNow do the replace-before-active test
7042026-10-22T23:31:16.749ZCompleted test: replace-before-active
7052026-10-22T23:31:16.749ZAll tests have completed, stopping all downstairs
7062026-10-22T23:31:16.766Z
7072026-10-22T23:31:16.770ZAll Tests have passed
7082026-10-22T23:31:16.771Z11:17 Test duration
7092026-10-22T23:31:16.771Z
7102026-10-22T23:31:16.771Zreal 11:17.062406239
7112026-10-22T23:31:16.771Zuser 15:38.739667861
7122026-10-22T23:31:16.771Zsys 11.626689505
7132026-10-22T23:31:16.771Ztrap 0.414159919
7142026-10-22T23:31:16.771Ztflt 0.036082073
7152026-10-22T23:31:16.771Zdflt 0.015446959
7162026-10-22T23:31:16.771Zkflt 0.001167799
7172026-10-22T23:31:16.771Zlock 15:13:50.273483098
7182026-10-22T23:31:16.771Zslp 48:11.996569424
7192026-10-22T23:31:16.771Zlat 1:48.423387772
7202026-10-22T23:31:16.772Zstop 0.040188054
7212026-10-22T23:31:16.772Z+ echo 'test-up-2region-encrypted ends'
7222026-10-22T23:31:16.772Ztest-up-2region-encrypted ends
7232026-10-22T23:31:21.770Zprocess exited: duration 687091 ms, exit code 0
7242026-10-22T23:31:21.770Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7252026-10-22T23:32:21.802Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7262026-10-22T23:32:21.802Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7272026-10-22T23:32:21.827Zfound 17 output files
7282026-10-22T23:32:21.827Zuploading: /tmp/test_up-build/dsc-out.txt (58177 bytes)
7292026-10-22T23:32:22.852Zuploaded: /tmp/test_up-build/dsc-out.txt
7302026-10-22T23:32:22.852Zuploading: /tmp/test_up-build/test_up_out.txt (23118504 bytes)
7312026-10-22T23:32:24.285Zuploaded: /tmp/test_up-build/test_up_out.txt
7322026-10-22T23:32:24.285Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (550977 bytes)
7332026-10-22T23:32:25.644Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7342026-10-22T23:32:25.644Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (548303 bytes)
7352026-10-22T23:32:26.771Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7362026-10-22T23:32:26.771Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (978684 bytes)
7372026-10-22T23:32:27.866Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7382026-10-22T23:32:27.866Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (543041 bytes)
7392026-10-22T23:32:28.944Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7402026-10-22T23:32:28.944Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (1343695 bytes)
7412026-10-22T23:32:30.048Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7422026-10-22T23:32:30.048Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (47344 bytes)
7432026-10-22T23:32:31.068Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7442026-10-22T23:32:31.069Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3578701 bytes)
7452026-10-22T23:32:32.195Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7462026-10-22T23:32:32.195Zuploading: /tmp/debug/df.txt (1270 bytes)
7472026-10-22T23:32:33.207Zuploaded: /tmp/debug/df.txt
7482026-10-22T23:32:33.208Zuploading: /tmp/debug/dtrace.txt (1043033 bytes)
7492026-10-22T23:32:34.281Zuploaded: /tmp/debug/dtrace.txt
7502026-10-22T23:32:34.281Zuploading: /tmp/debug/iostat.txt (223161 bytes)
7512026-10-22T23:32:34.309Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 223161 -> 226761
7522026-10-22T23:32:35.320Zuploaded: /tmp/debug/iostat.txt
7532026-10-22T23:32:35.320Zuploading: /tmp/debug/mpstat.txt (541832 bytes)
7542026-10-22T23:32:35.368Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 541832 -> 551310
7552026-10-22T23:32:36.384Zuploaded: /tmp/debug/mpstat.txt
7562026-10-22T23:32:36.384Zuploading: /tmp/debug/paging.txt (97295 bytes)
7572026-10-22T23:32:36.401Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 97295 -> 99063
7582026-10-22T23:32:37.412Zuploaded: /tmp/debug/paging.txt
7592026-10-22T23:32:37.412Zuploading: /tmp/debug/prstat.txt (1037459 bytes)
7602026-10-22T23:32:37.480Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1037459 -> 1056938
7612026-10-22T23:32:38.491Zuploaded: /tmp/debug/prstat.txt
7622026-10-22T23:32:38.491Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7632026-10-22T23:32:39.503Zuploaded: /tmp/debug/psrinfo.txt
7642026-10-22T23:32:39.503Zuploading: /tmp/debug/upstairs-info.txt (158270 bytes)
7652026-10-22T23:32:40.536Zuploaded: /tmp/debug/upstairs-info.txt