01KA0N9JS2A7PKVJZFXV42RFN4: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01KA0N9VZGWV6ETDR32581SXPA

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12025-11-14T08:04:09.754Zjob dependencies complete; ready to run (waiting for 17 m 11 s)
22025-11-14T08:04:40.848Zjob assigned to worker 01KA0P9CRN71C47C6984THK14A [factory gimlet-EVT22200007-propolis, EVT22200007/48525] (queued for 31 s)
32025-11-14T08:04:49.395Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42025-11-14T08:04:58.650Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52025-11-14T08:04:58.675Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62025-11-14T08:05:06.545Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72025-11-14T08:05:06.545Zdownloading input: /input/build/work/bins/crutest.gz
82025-11-14T08:05:19.920Zdownloaded input: /input/build/work/bins/crutest.gz
92025-11-14T08:05:19.920Zdownloading input: /input/build/work/bins/dsc.gz
102025-11-14T08:05:24.184Zdownloaded input: /input/build/work/bins/dsc.gz
112025-11-14T08:05:24.210Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122025-11-14T08:05:24.236Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132025-11-14T08:05:24.236Zdownloading input: /input/build/work/scripts/test_ds.sh
142025-11-14T08:05:24.269Zdownloaded input: /input/build/work/scripts/test_ds.sh
152025-11-14T08:05:24.269Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162025-11-14T08:05:24.310Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172025-11-14T08:05:24.310Zdownloading input: /input/build/work/scripts/test_repair.sh
182025-11-14T08:05:24.369Zdownloaded input: /input/build/work/scripts/test_repair.sh
192025-11-14T08:05:24.369Zdownloading input: /input/build/work/scripts/test_replay.sh
202025-11-14T08:05:24.397Zdownloaded input: /input/build/work/scripts/test_replay.sh
212025-11-14T08:05:24.397Zdownloading input: /input/build/work/scripts/test_up.sh
222025-11-14T08:05:24.441Zdownloaded input: /input/build/work/scripts/test_up.sh
232025-11-14T08:05:24.441Zdownloading input: /input/build/work/scripts/upstairs_info.d
242025-11-14T08:05:24.483Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252025-11-14T08:05:24.507Zdownloading input: /input/build/tmp/cargo-test-out.log
262025-11-14T08:05:28.407Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272025-11-14T08:05:28.407Zstarting task 0: "setup"
282025-11-14T08:05:28.432Z++ uname -s
292025-11-14T08:05:28.432Z+ kern=SunOS
302025-11-14T08:05:28.432Z+ build_user=build
312025-11-14T08:05:28.432Z+ build_uid=12345
322025-11-14T08:05:28.432Z+ work_dir=/work
332025-11-14T08:05:28.432Z+ input_dir=/input
342025-11-14T08:05:28.432Z+ [[ 0 == 12345 ]]
352025-11-14T08:05:28.432Z+ case "$kern" in
362025-11-14T08:05:28.432Z+ groupadd -g 12345 build
372025-11-14T08:05:28.432Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382025-11-14T08:05:30.432Z+ zfs create -o mountpoint=/work rpool/work
392025-11-14T08:05:30.469Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402025-11-14T08:05:30.495Z+ home_fs=zfs
412025-11-14T08:05:30.495Z+ [[ zfs == autofs ]]
422025-11-14T08:05:30.495Z+ mkdir -p /home/build
432025-11-14T08:05:30.495Z+ chown build:build /home/build /work
442025-11-14T08:05:32.479Z+ chmod 0700 /home/build /work
452025-11-14T08:05:32.504Zprocess exited: duration 4075 ms, exit code 0
 
462025-11-14T08:05:32.557Zstarting task 1: "authentication"
472025-11-14T08:05:32.602Zprocess exited: duration 47 ms, exit code 0
 
482025-11-14T08:05:32.651Zstarting task 2: "build"
492025-11-14T08:05:32.675Z+ banner cores
502025-11-14T08:05:32.676Z
512025-11-14T08:05:32.676Z #### #### ##### ###### ####
522025-11-14T08:05:32.676Z # # # # # # # #
532025-11-14T08:05:32.676Z # # # # # ##### ####
542025-11-14T08:05:32.676Z # # # ##### # #
552025-11-14T08:05:32.676Z # # # # # # # # #
562025-11-14T08:05:32.676Z #### #### # # ###### ####
572025-11-14T08:05:32.676Z
582025-11-14T08:05:32.676Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592025-11-14T08:05:32.676Z+ banner unpack
602025-11-14T08:05:32.676Z
612025-11-14T08:05:32.676Z # # # # ##### ## #### # #
622025-11-14T08:05:32.676Z # # ## # # # # # # # # #
632025-11-14T08:05:32.676Z # # # # # # # # # # ####
642025-11-14T08:05:32.676Z # # # # # ##### ###### # # #
652025-11-14T08:05:32.676Z # # # ## # # # # # # #
662025-11-14T08:05:32.676Z #### # # # # # #### # #
672025-11-14T08:05:32.676Z
682025-11-14T08:05:32.676Z+ mkdir -p /var/tmp/bins
692025-11-14T08:05:32.676Z+ for t in "$input/bins/"*.gz
702025-11-14T08:05:32.676Z++ basename /input/build/work/bins/crucible-downstairs.gz
712025-11-14T08:05:32.701Z+ b=crucible-downstairs.gz
722025-11-14T08:05:32.701Z+ b=crucible-downstairs
732025-11-14T08:05:32.701Z+ gunzip
742025-11-14T08:05:36.296Z+ chmod +x /var/tmp/bins/crucible-downstairs
752025-11-14T08:05:36.321Z+ for t in "$input/bins/"*.gz
762025-11-14T08:05:36.321Z++ basename /input/build/work/bins/crucible-hammer.gz
772025-11-14T08:05:36.321Z+ b=crucible-hammer.gz
782025-11-14T08:05:36.321Z+ b=crucible-hammer
792025-11-14T08:05:36.321Z+ gunzip
802025-11-14T08:05:39.848Z+ chmod +x /var/tmp/bins/crucible-hammer
812025-11-14T08:05:39.873Z+ for t in "$input/bins/"*.gz
822025-11-14T08:05:39.873Z++ basename /input/build/work/bins/crutest.gz
832025-11-14T08:05:39.873Z+ b=crutest.gz
842025-11-14T08:05:39.873Z+ b=crutest
852025-11-14T08:05:39.873Z+ gunzip
862025-11-14T08:05:43.223Z+ chmod +x /var/tmp/bins/crutest
872025-11-14T08:05:43.248Z+ for t in "$input/bins/"*.gz
882025-11-14T08:05:43.248Z++ basename /input/build/work/bins/dsc.gz
892025-11-14T08:05:43.248Z+ b=dsc.gz
902025-11-14T08:05:43.248Z+ b=dsc
912025-11-14T08:05:43.248Z+ gunzip
922025-11-14T08:05:44.985Z+ chmod +x /var/tmp/bins/dsc
932025-11-14T08:05:45.010Z+ export BINDIR=/var/tmp/bins
942025-11-14T08:05:45.010Z+ BINDIR=/var/tmp/bins
952025-11-14T08:05:45.010Z+ jobpid=1147
962025-11-14T08:05:45.010Z+ echo 'Setup debug logging'
972025-11-14T08:05:45.010Z+ mkdir /tmp/debug
982025-11-14T08:05:45.010ZSetup debug logging
992025-11-14T08:05:45.010Z+ sleep 3600
1002025-11-14T08:05:45.010Z+ psrinfo -v
1012025-11-14T08:05:45.010Z+ df -h
1022025-11-14T08:05:45.010Zdf: cannot statvfs /var/metadata/iso: Permission denied
1032025-11-14T08:05:45.036Z+ true
1042025-11-14T08:05:45.036Z+ prstat -d d -mLc 1
1052025-11-14T08:05:45.036Z+ iostat -T d -xn 1
1062025-11-14T08:05:45.036Z+ mpstat -T d 1
1072025-11-14T08:05:45.036Z+ vmstat -T d -p 1
1082025-11-14T08:05:45.036Z+ banner test_up_2r_encrypted
1092025-11-14T08:05:45.036Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102025-11-14T08:05:45.036Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1112025-11-14T08:05:45.036Z #####
1122025-11-14T08:05:45.036Z ##### ###### #### ##### # # ##### # # #####
1132025-11-14T08:05:45.036Z # # # # # # # # # # #
1142025-11-14T08:05:45.036Z # ##### #### # # # # # ##### # #
1152025-11-14T08:05:45.037Z # # # # # # ##### # #####
1162025-11-14T08:05:45.037Z # # # # # # # # # # #
1172025-11-14T08:05:45.037Z # ###### #### # ####### #### # ####### ####### # #
1182025-11-14T08:05:45.037Z
1192025-11-14T08:05:45.037Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1202025-11-14T08:05:45.064Z/input/build/work
1212025-11-14T08:05:45.089ZUsing 2 region sets
1222025-11-14T08:05:45.089ZTurn off color for downstairs dump
1232025-11-14T08:05:45.113ZUpstairs using key: WnXrRH79d9c3FgO1oyjgUT3Ii1EqU/M+koSgGiYGGGc=
1242025-11-14T08:05:45.113Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1252025-11-14T08:05:45.113ZCreating 6 downstairs regions
1262025-11-14T08:05:45.581ZStarting 6 downstairs
1272025-11-14T08:05:45.605Zdsc started at PID: 1195
1282025-11-14T08:05:51.072ZDisable automatic restart on all downstairs
1292025-11-14T08:05:51.149Z
1302025-11-14T08:05:51.149ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1312025-11-14T08:05:51.174ZRunning test: span
1322025-11-14T08:05:55.428ZCompleted test: span
1332025-11-14T08:06:00.438ZRunning test: big
1342025-11-14T08:06:05.203ZCompleted test: big
1352025-11-14T08:06:10.213ZRunning test: dep
1362025-11-14T08:06:47.094ZCompleted test: dep
1372025-11-14T08:06:52.103ZRunning test: balloon
1382025-11-14T08:11:55.762ZCompleted test: balloon
1392025-11-14T08:12:00.770ZRunning test: deactivate
1402025-11-14T08:12:05.631ZCompleted test: deactivate
1412025-11-14T08:12:10.640ZRunning hammer
1422025-11-14T08:12:26.534ZRun repair tests
1432025-11-14T08:12:26.558Z/var/tmp/bins/crutest fill -g 31 -q --key WnXrRH79d9c3FgO1oyjgUT3Ii1EqU/M+koSgGiYGGGc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1442025-11-14T08:12:26.612Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:25.999640552Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"task":"crutest"}
1452025-11-14T08:12:26.685Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.000496832Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"task":"crutest"}
1462025-11-14T08:12:26.685Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.003371586Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"task":"crutest"}
1472025-11-14T08:12:26.685Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.003660215Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
1482025-11-14T08:12:26.685Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"41d548fbbe70c1e1e65787dcf8bd6c02ea0af389\",\n git_commit_timestamp: \"2025-11-14T07:46:31.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.003826443Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
1492025-11-14T08:12:26.685Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.003961239Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
1502025-11-14T08:12:26.685Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.004088871Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
1512025-11-14T08:12:26.685Z{"msg":"Crucible 0feb7eb2-3600-493a-99e7-dd9f9bc65380 has session id: 38f07750-ad64-47ac-9a50-e067a18879cb","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.004924981Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1522025-11-14T08:12:26.685Z{"msg":"Upstairs opts: Upstairs UUID: 0feb7eb2-3600-493a-99e7-dd9f9bc65380, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.005049287Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1532025-11-14T08:12:26.685Z{"msg":"Crucible stats registered with UUID: 0feb7eb2-3600-493a-99e7-dd9f9bc65380","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.005127731Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1542025-11-14T08:12:26.685Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:26.005192459Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1552025-11-14T08:12:26.685Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.005414758Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1562025-11-14T08:12:26.685Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.005535396Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1572025-11-14T08:12:26.685Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.005626396Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1582025-11-14T08:12:26.685Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.008712787Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"task":"crutest"}
1592025-11-14T08:12:26.685Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.008830549Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
1602025-11-14T08:12:26.685Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"41d548fbbe70c1e1e65787dcf8bd6c02ea0af389\",\n git_commit_timestamp: \"2025-11-14T07:46:31.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.008918162Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
1612025-11-14T08:12:26.685Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.008981607Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
1622025-11-14T08:12:26.685Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.009064621Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
1632025-11-14T08:12:26.685Z{"msg":"Crucible 0feb7eb2-3600-493a-99e7-dd9f9bc65380 has session id: 098af754-bdad-40a8-9779-c0e547180fe5","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.009669244Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
1642025-11-14T08:12:26.685Z{"msg":"Upstairs opts: Upstairs UUID: 0feb7eb2-3600-493a-99e7-dd9f9bc65380, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.009772418Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
1652025-11-14T08:12:26.685Z{"msg":"Crucible stats registered with UUID: 0feb7eb2-3600-493a-99e7-dd9f9bc65380","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.009845943Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
1662025-11-14T08:12:26.685Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:26.009937624Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
1672025-11-14T08:12:26.685Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.010029205Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
1682025-11-14T08:12:26.685Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.010108922Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
1692025-11-14T08:12:26.685Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.01017906Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
1702025-11-14T08:12:26.685Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011385792Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"task":"crutest"}
1712025-11-14T08:12:26.685Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011463565Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
1722025-11-14T08:12:26.685Z{"msg":"0feb7eb2-3600-493a-99e7-dd9f9bc65380 active request set","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011539966Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1732025-11-14T08:12:26.685Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011604283Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"0","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1742025-11-14T08:12:26.685Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011669742Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"1","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1752025-11-14T08:12:26.686Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011746704Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"2","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1762025-11-14T08:12:26.686Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011790911Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1772025-11-14T08:12:26.686Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011829327Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1782025-11-14T08:12:26.686Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011865418Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1792025-11-14T08:12:26.686Z{"msg":"ds_connection connected from Ok(127.0.0.1:58950)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011900808Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1802025-11-14T08:12:26.686Z{"msg":"ds_connection connected from Ok(127.0.0.1:58942)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.011934585Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1812025-11-14T08:12:26.686Z{"msg":"ds_connection connected from Ok(127.0.0.1:36790)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.012174148Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1822025-11-14T08:12:26.686Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.013793423Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"1","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1832025-11-14T08:12:26.686Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014087935Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"0","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1842025-11-14T08:12:26.686Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014118295Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"2","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1852025-11-14T08:12:26.686Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014139436Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1862025-11-14T08:12:26.686Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014309111Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1872025-11-14T08:12:26.686Z{"msg":"[0]R flush_numbers: [3990, 3990, 3990, 3990, 3990]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014368629Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1882025-11-14T08:12:26.686Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014392315Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1892025-11-14T08:12:26.686Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014417705Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1902025-11-14T08:12:26.686Z{"msg":"[1]R flush_numbers: [3990, 3990, 3990, 3990, 3990]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014440941Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1912025-11-14T08:12:26.686Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014463035Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1922025-11-14T08:12:26.686Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.01450119Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1932025-11-14T08:12:26.686Z{"msg":"[2]R flush_numbers: [3990, 3990, 3990, 3990, 3990]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014523013Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1942025-11-14T08:12:26.686Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014543684Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1952025-11-14T08:12:26.686Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.01458192Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1962025-11-14T08:12:26.686Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014602069Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1972025-11-14T08:12:26.686Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014622039Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1982025-11-14T08:12:26.686Z{"msg":"Next flush: 3991","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014643742Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
1992025-11-14T08:12:26.686Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014681396Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
2002025-11-14T08:12:26.686Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014702518Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
2012025-11-14T08:12:26.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014737426Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"0","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
2022025-11-14T08:12:26.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014758268Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"1","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
2032025-11-14T08:12:26.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014780041Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"2","":"downstairs","session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
2042025-11-14T08:12:26.686Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014816523Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
2052025-11-14T08:12:26.686Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014839057Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
2062025-11-14T08:12:26.686Z{"msg":"0feb7eb2-3600-493a-99e7-dd9f9bc65380 is now active with session: 38f07750-ad64-47ac-9a50-e067a18879cb","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014861221Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
2072025-11-14T08:12:26.686Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014882773Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
2082025-11-14T08:12:26.686Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.014899276Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"38f07750-ad64-47ac-9a50-e067a18879cb"}
2092025-11-14T08:12:26.686Z{"msg":"Activated sub_volume 0feb7eb2-3600-493a-99e7-dd9f9bc65380","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015109581Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
2102025-11-14T08:12:26.686Z{"msg":"0feb7eb2-3600-493a-99e7-dd9f9bc65380 active request set","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015324906Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2112025-11-14T08:12:26.686Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015375436Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
2122025-11-14T08:12:26.686Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015395856Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"0","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2132025-11-14T08:12:26.687Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015416396Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"1","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2142025-11-14T08:12:26.687Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015436817Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"2","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2152025-11-14T08:12:26.687Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015485853Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2162025-11-14T08:12:26.687Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015511965Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2172025-11-14T08:12:26.687Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015609087Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2182025-11-14T08:12:26.687Z{"msg":"ds_connection connected from Ok(127.0.0.1:47387)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015652082Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2192025-11-14T08:12:26.687Z{"msg":"ds_connection connected from Ok(127.0.0.1:37008)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.015674045Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2202025-11-14T08:12:26.687Z{"msg":"ds_connection connected from Ok(127.0.0.1:54556)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.01602115Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2212025-11-14T08:12:26.687Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.016894604Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"0","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2222025-11-14T08:12:26.687Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.01697379Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"1","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2232025-11-14T08:12:26.687Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.017518074Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2242025-11-14T08:12:26.687Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.017595126Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2252025-11-14T08:12:26.687Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.017620146Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"2","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2262025-11-14T08:12:26.687Z{"msg":"[0]R flush_numbers: [5312, 5312, 5312, 5309, 5312]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.017953253Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2272025-11-14T08:12:26.687Z{"msg":"[0]R generation: [14, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018008572Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2282025-11-14T08:12:26.687Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018033531Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2292025-11-14T08:12:26.687Z{"msg":"[1]R flush_numbers: [5312, 5312, 5312, 5309, 5312]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018055094Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2302025-11-14T08:12:26.687Z{"msg":"[1]R generation: [14, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018158979Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2312025-11-14T08:12:26.687Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018185551Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2322025-11-14T08:12:26.687Z{"msg":"[2]R flush_numbers: [5312, 5312, 5312, 5309, 5312]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018206703Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2332025-11-14T08:12:26.687Z{"msg":"[2]R generation: [14, 14, 14, 12, 14]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018295428Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2342025-11-14T08:12:26.687Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018319055Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2352025-11-14T08:12:26.687Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018404684Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2362025-11-14T08:12:26.687Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.01842826Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2372025-11-14T08:12:26.687Z{"msg":"Next flush: 5313","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018502868Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2382025-11-14T08:12:26.687Z{"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2392025-11-14T08:12:26.687Z,"time":"2025-11-14T08:12:26.01854984Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"":"downstairs","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2402025-11-14T08:12:26.687Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018652503Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2412025-11-14T08:12:26.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018700317Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"0","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2422025-11-14T08:12:26.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018725136Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"1","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2432025-11-14T08:12:26.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018815374Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"client":"2","":"downstairs","session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2442025-11-14T08:12:26.687Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018838189Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2452025-11-14T08:12:26.712Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018857668Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2462025-11-14T08:12:26.712Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018929129Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
2472025-11-14T08:12:26.712Z{"msg":"0feb7eb2-3600-493a-99e7-dd9f9bc65380 is now active with session: 098af754-bdad-40a8-9779-c0e547180fe5","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.018954709Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2482025-11-14T08:12:26.712Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.019026952Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238,"session_id":"098af754-bdad-40a8-9779-c0e547180fe5"}
2492025-11-14T08:12:26.712Z{"msg":"Activated sub_volume 0feb7eb2-3600-493a-99e7-dd9f9bc65380","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:26.019064747Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
2502025-11-14T08:12:26.712ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2512025-11-14T08:12:26.713ZFill test
2522025-11-14T08:12:26.713ZRead and Verify all blocks (0..100 range:false)
2532025-11-14T08:12:26.777ZWrote out file "/var/tmp/test_up-build/verify_file"
2542025-11-14T08:12:26.777ZCLIENT: Tests done. All submitted work has been ACK'd
2552025-11-14T08:12:26.777Z----------------------------------------------------------------
2562025-11-14T08:12:26.777Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2572025-11-14T08:12:26.777ZStates: Active Active Active
2582025-11-14T08:12:26.777ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2592025-11-14T08:12:26.777Z 1002 Acked Read 50 Done Done Done false
2602025-11-14T08:12:26.777Z STATES DS:0 DS:1 DS:2 TOTAL
2612025-11-14T08:12:26.777Z Sent 0 0 0 0
2622025-11-14T08:12:26.777Z Done 1 1 1 3
2632025-11-14T08:12:26.777Z Skipped 0 0 0 0
2642025-11-14T08:12:26.777Z Error 0 0 0 0
2652025-11-14T08:12:26.777ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2662025-11-14T08:12:26.777ZDownstairs last five completed: 1001 1000
2672025-11-14T08:12:26.802ZUpstairs last five completed: 1002 1001 1000
2682025-11-14T08:12:26.802Z----------------------------------------------------------------
2692025-11-14T08:12:26.802Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2702025-11-14T08:12:26.802ZStates: Active Active Active
2712025-11-14T08:12:26.802ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2722025-11-14T08:12:26.802Z 1002 Acked Read 50 Done Done Done false
2732025-11-14T08:12:26.803Z STATES DS:0 DS:1 DS:2 TOTAL
2742025-11-14T08:12:26.803Z Sent 0 0 0 0
2752025-11-14T08:12:26.803Z Done 1 1 1 3
2762025-11-14T08:12:26.803Z Skipped 0 0 0 0
2772025-11-14T08:12:26.803Z Error 0 0 0 0
2782025-11-14T08:12:26.803ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2792025-11-14T08:12:26.803ZDownstairs last five completed: 1001 1000
2802025-11-14T08:12:26.803ZUpstairs last five completed: 1002 1001 1000
2812025-11-14T08:12:26.803ZCLIENT: Up:0 ds:2 act:6
2822025-11-14T08:12:30.779Z----------------------------------------------------------------
2832025-11-14T08:12:30.779Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2842025-11-14T08:12:30.779ZDownstairs last five completed: 1003 1002 1001 1000
2852025-11-14T08:12:30.779ZUpstairs last five completed: 1003 1002 1001 1000
2862025-11-14T08:12:30.804Z----------------------------------------------------------------
2872025-11-14T08:12:30.804Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2882025-11-14T08:12:30.804ZDownstairs last five completed: 1003 1002 1001 1000
2892025-11-14T08:12:30.804ZUpstairs last five completed: 1003 1002 1001 1000
2902025-11-14T08:12:30.805ZCLIENT: Up:0 ds:0 act:6
2912025-11-14T08:12:30.805ZCLIENT: All crucible jobs finished, exiting program
2922025-11-14T08:12:30.805Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:30.169057314Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1238}
2932025-11-14T08:12:30.805ZRepair setup passed
2942025-11-14T08:12:30.805ZCopy the region for /var/tmp/test_up-build/8830
2952025-11-14T08:12:30.805Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2962025-11-14T08:12:30.830Z/var/tmp/bins/crutest fill -g 32 -q --key WnXrRH79d9c3FgO1oyjgUT3Ii1EqU/M+koSgGiYGGGc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2972025-11-14T08:12:30.877Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.264789053Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"task":"crutest"}
2982025-11-14T08:12:30.947Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.26570972Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"task":"crutest"}
2992025-11-14T08:12:30.948Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.26829391Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"task":"crutest"}
3002025-11-14T08:12:30.948Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.268557751Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3012025-11-14T08:12:30.948Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"41d548fbbe70c1e1e65787dcf8bd6c02ea0af389\",\n git_commit_timestamp: \"2025-11-14T07:46:31.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.26862816Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3022025-11-14T08:12:30.948Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.268649652Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3032025-11-14T08:12:30.948Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.268694911Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3042025-11-14T08:12:30.948Z{"msg":"Crucible ba444c04-7839-4450-9a0a-8101b078d052 has session id: a15e71a2-f36a-4de6-acae-af1774f17488","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.269922875Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3052025-11-14T08:12:30.948Z{"msg":"Upstairs opts: Upstairs UUID: ba444c04-7839-4450-9a0a-8101b078d052, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.27001658Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3062025-11-14T08:12:30.948Z{"msg":"Crucible stats registered with UUID: ba444c04-7839-4450-9a0a-8101b078d052","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.270044735Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3072025-11-14T08:12:30.948Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:30.270065186Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3082025-11-14T08:12:30.948Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.270561124Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3092025-11-14T08:12:30.948Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.270628037Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3102025-11-14T08:12:30.948Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.270650882Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3112025-11-14T08:12:30.948Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.274143264Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"task":"crutest"}
3122025-11-14T08:12:30.948Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.274320514Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3132025-11-14T08:12:30.948Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"41d548fbbe70c1e1e65787dcf8bd6c02ea0af389\",\n git_commit_timestamp: \"2025-11-14T07:46:31.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.274398277Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3142025-11-14T08:12:30.948Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.274443015Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3152025-11-14T08:12:30.948Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.274501511Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3162025-11-14T08:12:30.948Z{"msg":"Crucible ba444c04-7839-4450-9a0a-8101b078d052 has session id: 51a0fc41-fca8-4308-8046-da8bcc667a75","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.275034433Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3172025-11-14T08:12:30.948Z{"msg":"Upstairs opts: Upstairs UUID: ba444c04-7839-4450-9a0a-8101b078d052, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.275117998Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3182025-11-14T08:12:30.948Z{"msg":"Crucible stats registered with UUID: ba444c04-7839-4450-9a0a-8101b078d052","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.275181553Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3192025-11-14T08:12:30.948Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:30.275236301Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3202025-11-14T08:12:30.949Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.275342841Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3212025-11-14T08:12:30.949Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.275405285Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3222025-11-14T08:12:30.949Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.27543326Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3232025-11-14T08:12:30.949Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.2768777Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"task":"crutest"}
3242025-11-14T08:12:30.949Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.276920104Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3252025-11-14T08:12:30.949Z{"msg":"ba444c04-7839-4450-9a0a-8101b078d052 active request set","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.277144307Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3262025-11-14T08:12:30.949Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.277229475Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"0","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3272025-11-14T08:12:30.949Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.277256809Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"1","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3282025-11-14T08:12:30.949Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.277289293Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"2","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3292025-11-14T08:12:30.949Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.277328189Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3302025-11-14T08:12:30.949Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.277377497Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3312025-11-14T08:12:30.949Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.27749568Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3322025-11-14T08:12:30.949Z{"msg":"ds_connection connected from Ok(127.0.0.1:42667)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.277562933Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3332025-11-14T08:12:30.949Z{"msg":"ds_connection connected from Ok(127.0.0.1:45570)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.277907002Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3342025-11-14T08:12:30.949Z{"msg":"ds_connection connected from Ok(127.0.0.1:59403)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.27799224Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3352025-11-14T08:12:30.949Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.279409236Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"1","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3362025-11-14T08:12:30.949Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.279468854Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"0","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3372025-11-14T08:12:30.949Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.279496479Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"2","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3382025-11-14T08:12:30.949Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.280202221Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3392025-11-14T08:12:30.949Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.280293391Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3402025-11-14T08:12:30.949Z{"msg":"[0]R flush_numbers: [3991, 3991, 3991, 3991, 3991]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.280450141Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3412025-11-14T08:12:30.949Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.28052596Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3422025-11-14T08:12:30.949Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.280595518Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3432025-11-14T08:12:30.949Z{"msg":"[1]R flush_numbers: [3991, 3991, 3991, 3991, 3991]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.28066249Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3442025-11-14T08:12:30.949Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.280726957Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3452025-11-14T08:12:30.950Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.28080428Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3462025-11-14T08:12:30.950Z{"msg":"[2]R flush_numbers: [3991, 3991, 3991, 3991, 3991]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.280869829Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3472025-11-14T08:12:30.950Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.28095133Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3482025-11-14T08:12:30.950Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281016789Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3492025-11-14T08:12:30.950Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281084272Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3502025-11-14T08:12:30.950Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281150453Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3512025-11-14T08:12:30.950Z{"msg":"Next flush: 3992","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281201714Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3522025-11-14T08:12:30.950Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281235581Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3532025-11-14T08:12:30.950Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.28128592Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3542025-11-14T08:12:30.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281320759Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"0","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3552025-11-14T08:12:30.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281363163Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"1","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3562025-11-14T08:12:30.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281419424Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"2","":"downstairs","session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3572025-11-14T08:12:30.950Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281454172Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3582025-11-14T08:12:30.950Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281575963Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3592025-11-14T08:12:30.950Z{"msg":"ba444c04-7839-4450-9a0a-8101b078d052 is now active with session: a15e71a2-f36a-4de6-acae-af1774f17488","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281647915Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3602025-11-14T08:12:30.950Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281684667Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"a15e71a2-f36a-4de6-acae-af1774f17488"}
3612025-11-14T08:12:30.950Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281818281Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3622025-11-14T08:12:30.950Z{"msg":"Activated sub_volume ba444c04-7839-4450-9a0a-8101b078d052","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281895343Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3632025-11-14T08:12:30.950Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.281958868Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
3642025-11-14T08:12:30.950Z{"msg":"ba444c04-7839-4450-9a0a-8101b078d052 active request set","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282093935Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3652025-11-14T08:12:30.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282146408Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"0","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3662025-11-14T08:12:30.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282190455Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"1","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3672025-11-14T08:12:30.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282252979Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"2","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3682025-11-14T08:12:30.950Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282290893Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3692025-11-14T08:12:30.950Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282340862Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3702025-11-14T08:12:30.951Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282418826Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3712025-11-14T08:12:30.951Z{"msg":"ds_connection connected from Ok(127.0.0.1:60072)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282500828Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3722025-11-14T08:12:30.951Z{"msg":"ds_connection connected from Ok(127.0.0.1:54307)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282589763Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3732025-11-14T08:12:30.951Z{"msg":"ds_connection connected from Ok(127.0.0.1:56814)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282658299Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3742025-11-14T08:12:30.951Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282811922Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"1","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3752025-11-14T08:12:30.951Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282881609Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"0","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3762025-11-14T08:12:30.951Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.282961517Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"2","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3772025-11-14T08:12:30.951Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.283467766Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3782025-11-14T08:12:30.951Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.2835559Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3792025-11-14T08:12:30.951Z{"msg":"[0]R flush_numbers: [5313, 5313, 5313, 5313, 5313]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.283623564Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3802025-11-14T08:12:30.951Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.28368748Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3812025-11-14T08:12:30.951Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.283751606Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3822025-11-14T08:12:30.951Z{"msg":"[1]R flush_numbers: [5313, 5313, 5313, 5313, 5313]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.283815262Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3832025-11-14T08:12:30.951Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.283879439Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3842025-11-14T08:12:30.951Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
3852025-11-14T08:12:30.951Ztime":"2025-11-14T08:12:30.283960058Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3862025-11-14T08:12:30.951Z{"msg":"[2]R flush_numbers: [5313, 5313, 5313, 5313, 5313]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.284091647Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3872025-11-14T08:12:30.951Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.284161175Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3882025-11-14T08:12:30.951Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.284227946Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3892025-11-14T08:12:30.951Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.284292223Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3902025-11-14T08:12:30.951Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.284357682Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3912025-11-14T08:12:30.951Z{"msg":"Next flush: 5314","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.284421238Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3922025-11-14T08:12:30.951Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.28449824Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"":"downstairs","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3932025-11-14T08:12:30.952Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.284578849Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3942025-11-14T08:12:30.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.28465529Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"0","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3952025-11-14T08:12:30.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
3962025-11-14T08:12:30.952Ztime":"2025-11-14T08:12:30.284746259Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"1","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3972025-11-14T08:12:30.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.284870524Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"client":"2","":"downstairs","session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3982025-11-14T08:12:30.977Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.284938058Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
3992025-11-14T08:12:30.977Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.284999489Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
4002025-11-14T08:12:30.977Z{"msg":"ba444c04-7839-4450-9a0a-8101b078d052 is now active with session: 51a0fc41-fca8-4308-8046-da8bcc667a75","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.285092242Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
4012025-11-14T08:12:30.977Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.285169715Z","hostname":"bmat-EVT22200007-0000bd8d",Loading write count information from file "/var/tmp/test_up-build/verify_file"
4022025-11-14T08:12:30.977Z"pid":1244,"session_id":"51a0fc41-fca8-4308-8046-da8bcc667a75"}
4032025-11-14T08:12:30.977Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30Fill test
4042025-11-14T08:12:30.977Z,"time":"2025-11-14T08:12:30.285358507Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
4052025-11-14T08:12:30.977Z{"msg":"Activated sub_volume ba444c04-7839-4450-9a0a-8101b078d052","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:30.285478705Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1244}
4062025-11-14T08:12:30.977ZRead and Verify all blocks (0..100 range:false)
4072025-11-14T08:12:31.042ZWrote out file "/var/tmp/test_up-build/verify_file"
4082025-11-14T08:12:31.042ZCLIENT: Tests done. All submitted work has been ACK'd
4092025-11-14T08:12:31.042Z----------------------------------------------------------------
4102025-11-14T08:12:31.042Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4112025-11-14T08:12:31.042ZStates: Active Active Active
4122025-11-14T08:12:31.042ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4132025-11-14T08:12:31.042Z 1002 Acked Read 50 Done Done Done false
4142025-11-14T08:12:31.042Z STATES DS:0 DS:1 DS:2 TOTAL
4152025-11-14T08:12:31.042Z Sent 0 0 0 0
4162025-11-14T08:12:31.042Z Done 1 1 1 3
4172025-11-14T08:12:31.042Z Skipped 0 0 0 0
4182025-11-14T08:12:31.042Z Error 0 0 0 0
4192025-11-14T08:12:31.042ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4202025-11-14T08:12:31.042ZDownstairs last five completed: 1001 1000
4212025-11-14T08:12:31.042ZUpstairs last five completed: 1002 1001 1000
4222025-11-14T08:12:31.067Z----------------------------------------------------------------
4232025-11-14T08:12:31.067Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4242025-11-14T08:12:31.067ZStates: Active Active Active
4252025-11-14T08:12:31.067ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4262025-11-14T08:12:31.067Z 1002 Acked Read 50 Done Done Done false
4272025-11-14T08:12:31.067Z STATES DS:0 DS:1 DS:2 TOTAL
4282025-11-14T08:12:31.067Z Sent 0 0 0 0
4292025-11-14T08:12:31.067Z Done 1 1 1 3
4302025-11-14T08:12:31.067Z Skipped 0 0 0 0
4312025-11-14T08:12:31.067Z Error 0 0 0 0
4322025-11-14T08:12:31.068ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4332025-11-14T08:12:31.068ZDownstairs last five completed: 1001 1000
4342025-11-14T08:12:31.068ZUpstairs last five completed: 1002 1001 1000
4352025-11-14T08:12:31.068ZCLIENT: Up:0 ds:2 act:6
4362025-11-14T08:12:35.044Z----------------------------------------------------------------
4372025-11-14T08:12:35.044Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4382025-11-14T08:12:35.044ZDownstairs last five completed: 1003 1002 1001 1000
4392025-11-14T08:12:35.044ZUpstairs last five completed: 1003 1002 1001 1000
4402025-11-14T08:12:35.044Z----------------------------------------------------------------
4412025-11-14T08:12:35.044Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4422025-11-14T08:12:35.044ZDownstairs last five completed: 1003 1002 1001 1000
4432025-11-14T08:12:35.044ZUpstairs last five completed: 1003 1002 1001 1000
4442025-11-14T08:12:35.044ZCLIENT: Up:0 ds:0 act:6
4452025-11-14T08:12:35.044ZCLIENT: All crucible jobs finished, exiting program
4462025-11-14T08:12:35.069ZRepair part 1 passed
4472025-11-14T08:12:35.069Z
4482025-11-14T08:12:35.069ZKill the current downstairs
4492025-11-14T08:12:35.265ZDownstairs 2 stopped
4502025-11-14T08:12:35.265Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4512025-11-14T08:12:35.297ZNow put back the original so we have a mismatch
4522025-11-14T08:12:35.297Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4532025-11-14T08:12:35.297ZRestart downstairs with old directory
4542025-11-14T08:12:35.337Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4552025-11-14T08:12:35.362ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4562025-11-14T08:12:35.362Z 0 000-009 32 32 31 3992 3992 3991 F F F <---
4572025-11-14T08:12:35.362Z 1 010-019 32 32 31 3992 3992 3991 F F F <---
4582025-11-14T08:12:35.362Z 2 020-029 32 32 31 3992 3992 3991 F F F <---
4592025-11-14T08:12:35.362Z 3 030-039 32 32 31 3992 3992 3991 F F F <---
4602025-11-14T08:12:35.362Z 4 040-049 32 32 31 3992 3992 3991 F F F <---
4612025-11-14T08:12:35.362ZMax gen: 32, Max flush: 3992
4622025-11-14T08:12:35.387ZError: Difference in extent metadata found!
4632025-11-14T08:12:35.387Zdump test found error as expected
4642025-11-14T08:12:35.387Z
4652025-11-14T08:12:35.387Z
4662025-11-14T08:12:35.387Z/var/tmp/bins/crutest verify --range -g 33 -q --key WnXrRH79d9c3FgO1oyjgUT3Ii1EqU/M+koSgGiYGGGc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
4672025-11-14T08:12:35.446Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.833851455Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"task":"crutest"}
4682025-11-14T08:12:35.493Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.834906708Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"task":"crutest"}
4692025-11-14T08:12:35.493Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.837482672Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"task":"crutest"}
4702025-11-14T08:12:35.493Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.837772353Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
4712025-11-14T08:12:35.493Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"41d548fbbe70c1e1e65787dcf8bd6c02ea0af389\",\n git_commit_timestamp: \"2025-11-14T07:46:31.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.837857391Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
4722025-11-14T08:12:35.493Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.837952739Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
4732025-11-14T08:12:35.493Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.838032026Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
4742025-11-14T08:12:35.493Z{"msg":"Crucible 85eb369f-dac6-461e-b649-f3b27b867e52 has session id: 8162b240-2715-409e-8b0c-db0bc16f292c","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.839289127Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4752025-11-14T08:12:35.493Z{"msg":"Upstairs opts: Upstairs UUID: 85eb369f-dac6-461e-b649-f3b27b867e52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.839492408Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4762025-11-14T08:12:35.493Z{"msg":"Crucible stats registered with UUID: 85eb369f-dac6-461e-b649-f3b27b867e52","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.839670249Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4772025-11-14T08:12:35.493Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:34.839815485Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4782025-11-14T08:12:35.493Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.839956945Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4792025-11-14T08:12:35.493Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.840119065Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4802025-11-14T08:12:35.493Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.840147932Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4812025-11-14T08:12:35.493Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.843082824Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"task":"crutest"}
4822025-11-14T08:12:35.493Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.84325812Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
4832025-11-14T08:12:35.494Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"41d548fbbe70c1e1e65787dcf8bd6c02ea0af389\",\n git_commit_timestamp: \"2025-11-14T07:46:31.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.843420801Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
4842025-11-14T08:12:35.494Z{"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.843472102Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
4852025-11-14T08:12:35.494Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.84350147Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
4862025-11-14T08:12:35.494Z{"msg":"Crucible 85eb369f-dac6-461e-b649-f3b27b867e52 has session id: f868c8bb-e3fa-496e-af4a-e64aefc44994","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.844386817Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
4872025-11-14T08:12:35.494Z{"msg":"Upstairs opts: Upstairs UUID: 85eb369f-dac6-461e-b649-f3b27b867e52, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.844497455Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
4882025-11-14T08:12:35.494Z{"msg":"Crucible stats registered with UUID: 85eb369f-dac6-461e-b649-f3b27b867e52","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.844644886Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
4892025-11-14T08:12:35.494Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:34.844767037Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
4902025-11-14T08:12:35.494Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.844891102Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
4912025-11-14T08:12:35.494Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.845003053Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
4922025-11-14T08:12:35.494Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.845152448Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
4932025-11-14T08:12:35.494Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.845917087Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"task":"crutest"}
4942025-11-14T08:12:35.494Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.845996814Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
4952025-11-14T08:12:35.494Z{"msg":"85eb369f-dac6-461e-b649-f3b27b867e52 active request set","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846082293Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4962025-11-14T08:12:35.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846124537Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"0","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4972025-11-14T08:12:35.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846198743Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"1","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4982025-11-14T08:12:35.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846241357Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
4992025-11-14T08:12:35.494Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846299021Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5002025-11-14T08:12:35.494Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846375352Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5012025-11-14T08:12:35.494Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846435921Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5022025-11-14T08:12:35.494Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:34.84646578Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5032025-11-14T08:12:35.494Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:34.846494607Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5042025-11-14T08:12:35.494Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:34.846523734Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5052025-11-14T08:12:35.494Z{"msg":"ds_connection connected from Ok(127.0.0.1:35814)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846544806Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5062025-11-14T08:12:35.494Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846615596Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5072025-11-14T08:12:35.494Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846654132Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5082025-11-14T08:12:35.494Z{"msg":"ds_connection connected from Ok(127.0.0.1:48162)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.846684071Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5092025-11-14T08:12:35.494Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.847885732Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"1","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5102025-11-14T08:12:35.494Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.848028033Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"0","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5112025-11-14T08:12:35.494Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.848493692Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5122025-11-14T08:12:35.494Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:34.848534242Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5132025-11-14T08:12:45.459Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:44.847462155Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5142025-11-14T08:12:45.459Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:44.847613303Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5152025-11-14T08:12:45.459Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:44.847664965Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5162025-11-14T08:12:45.459Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:44.847710094Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5172025-11-14T08:12:45.459Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:44.847767928Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5182025-11-14T08:12:45.459Z{"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:44.847808849Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5192025-11-14T08:12:55.460Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.848991572Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5202025-11-14T08:12:55.461Z{"msg":"ds_connection connected from Ok(127.0.0.1:44585)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.849121999Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5212025-11-14T08:12:55.530Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.851908158Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5222025-11-14T08:12:55.530Z{"msg":"[0]R flush_numbers: [3992, 3992, 3992, 3992, 3992]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853155179Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5232025-11-14T08:12:55.530Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853251609Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5242025-11-14T08:12:55.530Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853277781Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5252025-11-14T08:12:55.530Z{"msg":"[1]R flush_numbers: [3992, 3992, 3992, 3992, 3992]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853301588Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5262025-11-14T08:12:55.530Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853337779Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5272025-11-14T08:12:55.530Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853364182Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5282025-11-14T08:12:55.530Z{"msg":"[2]R flush_numbers: [3991, 3991, 3991, 3991, 3991]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853388409Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5292025-11-14T08:12:55.530Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853425533Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5302025-11-14T08:12:55.530Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853448117Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5312025-11-14T08:12:55.530Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853482545Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5322025-11-14T08:12:55.530Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.85350521Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5332025-11-14T08:12:55.530Z{"msg":"Next flush: 3993","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853540189Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5342025-11-14T08:12:55.530Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3991, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853574156Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5352025-11-14T08:12:55.530Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3991, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853624375Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5362025-11-14T08:12:55.530Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853663853Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5372025-11-14T08:12:55.530Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3991, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.85369787Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5382025-11-14T08:12:55.530Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3991, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853730434Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5392025-11-14T08:12:55.530Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.85376891Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5402025-11-14T08:12:55.530Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3991, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853802877Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5412025-11-14T08:12:55.530Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3991, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853849469Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5422025-11-14T08:12:55.530Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853889879Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5432025-11-14T08:12:55.530Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3991, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.853937663Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5442025-11-14T08:12:55.530Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3991, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.85399151Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5452025-11-14T08:12:55.530Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854016499Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5462025-11-14T08:12:55.530Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3991, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854063922Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5472025-11-14T08:12:55.530Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3992, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3991, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854095785Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5482025-11-14T08:12:55.530Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854132077Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"mend","":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5492025-11-14T08:12:55.530Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854173599Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"0","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5502025-11-14T08:12:55.530Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854211213Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"1","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5512025-11-14T08:12:55.530Z{"msg":"Transition from Connecting { state: WaitQuorum, mode: New } to Reconcile","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854233948Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5522025-11-14T08:12:55.530Z{"msg":"Full repair list: {ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854284838Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5532025-11-14T08:12:55.531Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 3993, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 3993, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 3993, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 3993, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 3993, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854552657Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5542025-11-14T08:12:55.531Z{"msg":"starting reconciliation fceeb6f2-362e-4302-a8fd-3afff7781378: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854580542Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5552025-11-14T08:12:55.531Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.854601984Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5562025-11-14T08:12:55.531Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.863253829Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5572025-11-14T08:12:55.531Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.869127791Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5582025-11-14T08:12:55.557Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.875362205Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5592025-11-14T08:12:55.557Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.880814385Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5602025-11-14T08:12:55.557Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.886393856Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5612025-11-14T08:12:55.557Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.886451009Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"0","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5622025-11-14T08:12:55.557Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.88647685Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"1","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5632025-11-14T08:12:55.557Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.886501068Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5642025-11-14T08:12:55.557Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.886525226Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5652025-11-14T08:12:55.557Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.886560014Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5662025-11-14T08:12:55.557Z{"msg":"85eb369f-dac6-461e-b649-f3b27b867e52 is now active with session: 8162b240-2715-409e-8b0c-db0bc16f292c","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.88658324Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
5672025-11-14T08:12:55.557Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.886617047Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
5682025-11-14T08:12:55.557Z{"msg":"Activated sub_volume 85eb369f-dac6-461e-b649-f3b27b867e52","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.886771612Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
5692025-11-14T08:12:55.557Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.886946788Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
5702025-11-14T08:12:55.557Z{"msg":"85eb369f-dac6-461e-b649-f3b27b867e52 active request set","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.886982889Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5712025-11-14T08:12:55.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.8870035Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"0","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5722025-11-14T08:12:55.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.887025222Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"1","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5732025-11-14T08:12:55.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.887061073Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5742025-11-14T08:12:55.557Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.887082265Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5752025-11-14T08:12:55.557Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.887104329Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5762025-11-14T08:12:55.558Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.887125901Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5772025-11-14T08:12:55.558Z{"msg":"ds_connection connected from Ok(127.0.0.1:34823)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.887152995Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5782025-11-14T08:12:55.558Z{"msg":"ds_connection connected from Ok(127.0.0.1:38672)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.887465391Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5792025-11-14T08:12:55.558Z{"msg":"ds_connection connected from Ok(127.0.0.1:42585)","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.887551932Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5802025-11-14T08:12:55.558Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.888593307Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"1","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5812025-11-14T08:12:55.558Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.888796448Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"0","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5822025-11-14T08:12:55.558Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.888988958Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5832025-11-14T08:12:55.558Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889031562Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5842025-11-14T08:12:55.558Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889153863Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5852025-11-14T08:12:55.558Z{"msg":"[0]R flush_numbers: [5314, 5314, 5314, 5314, 5314]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889396683Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5862025-11-14T08:12:55.558Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889436571Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5872025-11-14T08:12:55.558Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889458705Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5882025-11-14T08:12:55.558Z{"msg":"[1]R flush_numbers: [5314, 5314, 5314, 5314, 5314]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889483043Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5892025-11-14T08:12:55.558Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889518683Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5902025-11-14T08:12:55.558Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889554384Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5912025-11-14T08:12:55.558Z{"msg":"[2]R flush_numbers: [5314, 5314, 5314, 5314, 5314]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889576267Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5922025-11-14T08:12:55.558Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889610444Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5932025-11-14T08:12:55.558Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889631276Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5942025-11-14T08:12:55.558Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889653309Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5952025-11-14T08:12:55.558Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.88967382Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5962025-11-14T08:12:55.558Z{"msg":"Next flush: 5315","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889707486Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5972025-11-14T08:12:55.558Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889727355Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"downstairs","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5982025-11-14T08:12:55.558Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889746954Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
5992025-11-14T08:12:55.558Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889764989Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"0","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
6002025-11-14T08:12:55.558Zf868c8bb-e3fa-496e-af4a-e64aefc44994"}
6012025-11-14T08:12:55.558Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889806351Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"1","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
6022025-11-14T08:12:55.558Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889826451Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"client":"2","":"downstairs","session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
6032025-11-14T08:12:55.558Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.88986173Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
6042025-11-14T08:12:55.558Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889882291Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
6052025-11-14T08:12:55.558Z{"msg":"85eb369f-dac6-461e-b649-f3b27b867e52 is now active with session: f868c8bb-e3fa-496e-af4a-e64aefc44994","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889918462Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
6062025-11-14T08:12:55.558Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.88993737Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"session_id":"f868c8bb-e3fa-496e-af4a-e64aefc44994"}
6072025-11-14T08:12:55.558Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889971738Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
6082025-11-14T08:12:55.558Z{"msg":"Activated sub_volume 85eb369f-dac6-461e-b649-f3b27b867e52","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:54.889988721Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
6092025-11-14T08:12:55.558ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6102025-11-14T08:12:55.558ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6112025-11-14T08:12:55.558ZRead and Verify all blocks (0..100 range:true)
6122025-11-14T08:12:55.597ZWrote out file "/var/tmp/test_up-build/verify_file"
6132025-11-14T08:12:55.597ZVerify test completed
6142025-11-14T08:12:55.621ZWrote out file "/var/tmp/test_up-build/verify_file"
6152025-11-14T08:12:55.621ZCLIENT: Tests done. All submitted work has been ACK'd
6162025-11-14T08:12:55.621Z----------------------------------------------------------------
6172025-11-14T08:12:55.621Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6182025-11-14T08:12:55.621ZStates: Active Active Active
6192025-11-14T08:12:55.621ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6202025-11-14T08:12:55.622Z 1000 Acked Read 50 Done Done Done false
6212025-11-14T08:12:55.622Z STATES DS:0 DS:1 DS:2 TOTAL
6222025-11-14T08:12:55.622Z Sent 0 0 0 0
6232025-11-14T08:12:55.622Z Done 1 1 1 3
6242025-11-14T08:12:55.622Z Skipped 0 0 0 0
6252025-11-14T08:12:55.622Z Error 0 0 0 0
6262025-11-14T08:12:55.622ZLast Flush: None None None
6272025-11-14T08:12:55.622ZDownstairs last five completed:
6282025-11-14T08:12:55.622ZUpstairs last five completed: 1000
6292025-11-14T08:12:55.622Z----------------------------------------------------------------
6302025-11-14T08:12:55.622Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6312025-11-14T08:12:55.622ZStates: Active Active Active
6322025-11-14T08:12:55.622ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6332025-11-14T08:12:55.622Z 1000 Acked Read 50 Done Done Done false
6342025-11-14T08:12:55.622Z STATES DS:0 DS:1 DS:2 TOTAL
6352025-11-14T08:12:55.622Z Sent 0 0 0 0
6362025-11-14T08:12:55.622Z Done 1 1 1 3
6372025-11-14T08:12:55.622Z Skipped 0 0 0 0
6382025-11-14T08:12:55.622Z Error 0 0 0 0
6392025-11-14T08:12:55.622ZLast Flush: None None None
6402025-11-14T08:12:55.622ZDownstairs last five completed:
6412025-11-14T08:12:55.622ZUpstairs last five completed: 1000
6422025-11-14T08:12:55.622ZCLIENT: Up:0 ds:2 act:6
6432025-11-14T08:12:59.599Z----------------------------------------------------------------
6442025-11-14T08:12:59.599Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6452025-11-14T08:12:59.600ZDownstairs last five completed: 1001 1000
6462025-11-14T08:12:59.600ZUpstairs last five completed: 1001 1000
6472025-11-14T08:12:59.624Z----------------------------------------------------------------
6482025-11-14T08:12:59.624Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6492025-11-14T08:12:59.624ZDownstairs last five completed: 1001 1000
6502025-11-14T08:12:59.624ZUpstairs last five completed: 1001 1000
6512025-11-14T08:12:59.624ZCLIENT: Up:0 ds:0 act:6
6522025-11-14T08:12:59.624ZCLIENT: All crucible jobs finished, exiting program
6532025-11-14T08:12:59.625Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:58.988645505Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252}
6542025-11-14T08:12:59.625Z{"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:58.988713139Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
6552025-11-14T08:12:59.625Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:58.98874936Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
6562025-11-14T08:12:59.625Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2025-11-14T08:12:58.988773388Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
6572025-11-14T08:12:59.625Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-11-14T08:12:58.988801173Z","hostname":"bmat-EVT22200007-0000bd8d","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"8162b240-2715-409e-8b0c-db0bc16f292c"}
6582025-11-14T08:12:59.625ZRepair part 2 passed
6592025-11-14T08:12:59.625Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
6602025-11-14T08:12:59.649ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6612025-11-14T08:12:59.649Z 0 000-009 32 32 32 3992 3992 3992 F F F
6622025-11-14T08:12:59.649Z 1 010-019 32 32 32 3992 3992 3992 F F F
6632025-11-14T08:12:59.649Z 2 020-029 32 32 32 3992 3992 3992 F F F
6642025-11-14T08:12:59.649Z 3 030-039 32 32 32 3992 3992 3992 F F F
6652025-11-14T08:12:59.650Z 4 040-049 32 32 32 3992 3992 3992 F F F
6662025-11-14T08:12:59.650ZMax gen: 32, Max flush: 3992
6672025-11-14T08:12:59.650Zdump test passed
6682025-11-14T08:12:59.650Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
6692025-11-14T08:12:59.674Z Extent 1
6702025-11-14T08:12:59.674ZGEN 32 32 32
6712025-11-14T08:12:59.674ZFLUSH_ID 3992 3992 3992
6722025-11-14T08:12:59.674ZDIRTY
6732025-11-14T08:12:59.674Z
6742025-11-14T08:12:59.674ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
6752025-11-14T08:12:59.674Z010 A A A A A A
6762025-11-14T08:12:59.674Z011 A A A A A A
6772025-11-14T08:12:59.674Z012 A A A A A A
6782025-11-14T08:12:59.674Z013 A A A A A A
6792025-11-14T08:12:59.674Z014 A A A A A A
6802025-11-14T08:12:59.674Z015 A A A A A A
6812025-11-14T08:12:59.675Z016 A A A A A A
6822025-11-14T08:12:59.699Z017 A A A A A A
6832025-11-14T08:12:59.699Z018 A A A A A A
6842025-11-14T08:12:59.699Z019 A A A A A A
6852025-11-14T08:12:59.699Zdump extent test passed
6862025-11-14T08:12:59.699Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
6872025-11-14T08:12:59.724ZExtent 2 Block in extent 0 Actual block 20
6882025-11-14T08:12:59.724Z
6892025-11-14T08:12:59.724Z DATA SHA256 VER
6902025-11-14T08:12:59.724Z------ ---------------------------------------------------------------- ---
6912025-11-14T08:12:59.724Z 0 15ab2eec0dada280823fb8d3421bfa430638c6a651245634bf35016bc5d7e487 A
6922025-11-14T08:12:59.724Z 1 15ab2eec0dada280823fb8d3421bfa430638c6a651245634bf35016bc5d7e487 A
6932025-11-14T08:12:59.724Z 2 15ab2eec0dada280823fb8d3421bfa430638c6a651245634bf35016bc5d7e487 A
6942025-11-14T08:12:59.724Z
6952025-11-14T08:12:59.724ZNONCES 0 1 2 DIFF
6962025-11-14T08:12:59.724Z------ ------------------------ ------------------------ ------------------------ -----
6972025-11-14T08:12:59.724Z 0 86ee79f553fc3e88f9a1aa51 86ee79f553fc3e88f9a1aa51 86ee79f553fc3e88f9a1aa51
6982025-11-14T08:12:59.724Z
6992025-11-14T08:12:59.724Z TAGS 0 1 2 DIFF
7002025-11-14T08:12:59.724Z------ -------------------------------- -------------------------------- -------------------------------- -----
7012025-11-14T08:12:59.724Z 0 da333d9fc3a2e508fe075c33e9273e1c da333d9fc3a2e508fe075c33e9273e1c da333d9fc3a2e508fe075c33e9273e1c
7022025-11-14T08:12:59.724Z
7032025-11-14T08:12:59.724ZHASHES 0 1 2 DIFF
7042025-11-14T08:12:59.724Z------ ---------------- ---------------- ---------------- -----
7052025-11-14T08:12:59.724Z
7062025-11-14T08:12:59.724Zdump block test passed
7072025-11-14T08:12:59.724ZInitial upstairs tests have completed, stopping all downstairs
7082025-11-14T08:13:04.799ZCreating 7 larger downstairs regions
7092025-11-14T08:13:05.340ZStarting 7 downstairs
7102025-11-14T08:13:10.353Zdsc restarted at PID: 1268
7112025-11-14T08:13:10.878ZNow do the replace-reconcile test
7122025-11-14T08:13:10.955ZUsing 8870 for the replacement port
7132025-11-14T08:16:39.256ZCompleted test: replace-reconcile
7142025-11-14T08:16:39.256ZNow do the replace-before-active test
7152025-11-14T08:21:43.054ZCompleted test: replace-before-active
7162025-11-14T08:21:43.078ZAll tests have completed, stopping all downstairs
7172025-11-14T08:21:43.135Z
7182025-11-14T08:21:43.135ZAll Tests have passed
7192025-11-14T08:21:43.160Z15:58 Test duration
7202025-11-14T08:21:43.160Z
7212025-11-14T08:21:43.160Zreal 15:58.098132133
7222025-11-14T08:21:43.160Zuser 32:57.451248586
7232025-11-14T08:21:43.160Zsys 50.061602265
7242025-11-14T08:21:43.160Ztrap 1.085132453
7252025-11-14T08:21:43.160Ztflt 0.091344204
7262025-11-14T08:21:43.160Zdflt 0.167533714
7272025-11-14T08:21:43.160Zkflt 0.001446325
7282025-11-14T08:21:43.160Zlock 26:56:30.633988974
7292025-11-14T08:21:43.160Zslp 1:20:02.966711659
7302025-11-14T08:21:43.160Zlat 4:31.528325207
7312025-11-14T08:21:43.160Zstop 0.153923098
7322025-11-14T08:21:43.160Z+ echo 'test-up-2region-encrypted ends'
7332025-11-14T08:21:43.160Ztest-up-2region-encrypted ends
7342025-11-14T08:21:48.145Zprocess exited: duration 970492 ms, exit code 0
7352025-11-14T08:21:48.145Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7362025-11-14T08:22:48.190Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7372025-11-14T08:22:48.190Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7382025-11-14T08:22:48.269Zfound 17 output files
7392025-11-14T08:22:48.269Zuploading: /tmp/test_up-build/dsc-out.txt (54019 bytes)
7402025-11-14T08:22:49.433Zuploaded: /tmp/test_up-build/dsc-out.txt
7412025-11-14T08:22:49.433Zuploading: /tmp/test_up-build/test_up_out.txt (20046577 bytes)
7422025-11-14T08:22:51.682Zuploaded: /tmp/test_up-build/test_up_out.txt
7432025-11-14T08:22:51.682Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (543622 bytes)
7442025-11-14T08:22:52.940Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7452025-11-14T08:22:52.965Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (540992 bytes)
7462025-11-14T08:22:54.200Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7472025-11-14T08:22:54.200Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (968991 bytes)
7482025-11-14T08:22:55.482Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7492025-11-14T08:22:55.482Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (535794 bytes)
7502025-11-14T08:22:56.738Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7512025-11-14T08:22:56.762Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (2467 bytes)
7522025-11-14T08:22:57.834Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7532025-11-14T08:22:57.835Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (38914 bytes)
7542025-11-14T08:22:58.999Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7552025-11-14T08:22:59.025Zuploading: /tmp/test_up-build/dsc/downstairs-8870.txt (3565113 bytes)
7562025-11-14T08:23:00.343Zuploaded: /tmp/test_up-build/dsc/downstairs-8870.txt
7572025-11-14T08:23:00.343Zuploading: /tmp/debug/df.txt (1270 bytes)
7582025-11-14T08:23:01.440Zuploaded: /tmp/debug/df.txt
7592025-11-14T08:23:01.440Zuploading: /tmp/debug/dtrace.txt (1415636 bytes)
7602025-11-14T08:23:02.728Zuploaded: /tmp/debug/dtrace.txt
7612025-11-14T08:23:02.728Zuploading: /tmp/debug/iostat.txt (377999 bytes)
7622025-11-14T08:23:02.914Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 377999 -> 383165
7632025-11-14T08:23:03.986Zuploaded: /tmp/debug/iostat.txt
7642025-11-14T08:23:03.986Zuploading: /tmp/debug/mpstat.txt (747541 bytes)
7652025-11-14T08:23:04.225Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 747541 -> 759221
7662025-11-14T08:23:05.297Zuploaded: /tmp/debug/mpstat.txt
7672025-11-14T08:23:05.297Zuploading: /tmp/debug/paging.txt (135058 bytes)
7682025-11-14T08:23:05.459Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 135058 -> 137346
7692025-11-14T08:23:06.531Zuploaded: /tmp/debug/paging.txt
7702025-11-14T08:23:06.531Zuploading: /tmp/debug/prstat.txt (1482193 bytes)
7712025-11-14T08:23:07.327Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 1482193 -> 1506756
7722025-11-14T08:23:08.398Zuploaded: /tmp/debug/prstat.txt
7732025-11-14T08:23:08.398Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
7742025-11-14T08:23:09.494Zuploaded: /tmp/debug/psrinfo.txt
7752025-11-14T08:23:09.494Zuploading: /tmp/debug/upstairs-info.txt (231574 bytes)
7762025-11-14T08:23:11.124Zuploaded: /tmp/debug/upstairs-info.txt