01M6P9D2WSPKNZPGXRAN0MB00F: test-up-2region-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-3.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01M6P9DJ0CNAX5FPNYWPKMS3ER

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12026-11-05T16:54:03.989Zjob dependencies complete; ready to run (waiting for 11 m 58 s)
22026-11-05T16:55:04.258Zjob assigned to worker 01M6PA3NDJFXMAXPV7J39FA428 [factory aws, i-03e130c25c0731151] (queued for 1 m 0 s)
32026-11-05T16:55:11.563Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42026-11-05T16:55:16.825Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52026-11-05T16:55:16.872Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62026-11-05T16:55:19.830Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72026-11-05T16:55:19.833Zdownloading input: /input/build/work/bins/crutest.gz
82026-11-05T16:55:21.871Zdownloaded input: /input/build/work/bins/crutest.gz
92026-11-05T16:55:21.871Zdownloading input: /input/build/work/bins/dsc.gz
102026-11-05T16:55:22.751Zdownloaded input: /input/build/work/bins/dsc.gz
112026-11-05T16:55:22.760Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122026-11-05T16:55:22.792Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132026-11-05T16:55:22.796Zdownloading input: /input/build/work/scripts/test_ds.sh
142026-11-05T16:55:22.814Zdownloaded input: /input/build/work/scripts/test_ds.sh
152026-11-05T16:55:22.814Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162026-11-05T16:55:22.844Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172026-11-05T16:55:22.848Zdownloading input: /input/build/work/scripts/test_repair.sh
182026-11-05T16:55:22.876Zdownloaded input: /input/build/work/scripts/test_repair.sh
192026-11-05T16:55:22.879Zdownloading input: /input/build/work/scripts/test_replay.sh
202026-11-05T16:55:22.906Zdownloaded input: /input/build/work/scripts/test_replay.sh
212026-11-05T16:55:22.906Zdownloading input: /input/build/work/scripts/test_up.sh
222026-11-05T16:55:22.950Zdownloaded input: /input/build/work/scripts/test_up.sh
232026-11-05T16:55:22.950Zdownloading input: /input/build/work/scripts/upstairs_info.d
242026-11-05T16:55:22.981Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252026-11-05T16:55:22.985Zdownloading input: /input/build/tmp/cargo-test-out.log
262026-11-05T16:55:23.797Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272026-11-05T16:55:23.797Zstarting task 0: "setup"
282026-11-05T16:55:23.802Z++ uname -s
292026-11-05T16:55:23.806Z+ kern=SunOS
302026-11-05T16:55:23.806Z+ build_user=build
312026-11-05T16:55:23.806Z+ build_uid=12345
322026-11-05T16:55:23.806Z+ work_dir=/work
332026-11-05T16:55:23.806Z+ input_dir=/input
342026-11-05T16:55:23.806Z+ [[ 0 == 12345 ]]
352026-11-05T16:55:23.806Z+ case "$kern" in
362026-11-05T16:55:23.806Z+ groupadd -g 12345 build
372026-11-05T16:55:23.807Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
382026-11-05T16:55:25.812Z+ zfs create -o mountpoint=/work rpool/work
392026-11-05T16:55:26.183Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
402026-11-05T16:55:26.187Z+ home_fs=zfs
412026-11-05T16:55:26.187Z+ [[ zfs == autofs ]]
422026-11-05T16:55:26.187Z+ mkdir -p /home/build
432026-11-05T16:55:26.187Z+ chown build:build /home/build /work
442026-11-05T16:55:27.187Z+ chmod 0700 /home/build /work
452026-11-05T16:55:27.192Zprocess exited: duration 3393 ms, exit code 0
 
462026-11-05T16:55:27.200Zstarting task 1: "authentication"
472026-11-05T16:55:27.218Zprocess exited: duration 18 ms, exit code 0
 
482026-11-05T16:55:27.225Zstarting task 2: "build"
492026-11-05T16:55:27.229Z+ banner cores
502026-11-05T16:55:27.233Z
512026-11-05T16:55:27.233Z #### #### ##### ###### ####
522026-11-05T16:55:27.233Z # # # # # # # #
532026-11-05T16:55:27.233Z # # # # # ##### ####
542026-11-05T16:55:27.233Z # # # ##### # #
552026-11-05T16:55:27.233Z # # # # # # # # #
562026-11-05T16:55:27.233Z #### #### # # ###### ####
572026-11-05T16:55:27.233Z
582026-11-05T16:55:27.233Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
592026-11-05T16:55:27.237Z+ banner unpack
602026-11-05T16:55:27.240Z
612026-11-05T16:55:27.241Z # # # # ##### ## #### # #
622026-11-05T16:55:27.241Z # # ## # # # # # # # # #
632026-11-05T16:55:27.241Z # # # # # # # # # # ####
642026-11-05T16:55:27.241Z # # # # # ##### ###### # # #
652026-11-05T16:55:27.241Z # # # ## # # # # # # #
662026-11-05T16:55:27.241Z #### # # # # # #### # #
672026-11-05T16:55:27.241Z
682026-11-05T16:55:27.241Z+ mkdir -p /var/tmp/bins
692026-11-05T16:55:27.241Z+ for t in "$input/bins/"*.gz
702026-11-05T16:55:27.241Z++ basename /input/build/work/bins/crucible-downstairs.gz
712026-11-05T16:55:27.245Z+ b=crucible-downstairs.gz
722026-11-05T16:55:27.245Z+ b=crucible-downstairs
732026-11-05T16:55:27.245Z+ gunzip
742026-11-05T16:55:29.081Z+ chmod +x /var/tmp/bins/crucible-downstairs
752026-11-05T16:55:29.085Z+ for t in "$input/bins/"*.gz
762026-11-05T16:55:29.085Z++ basename /input/build/work/bins/crucible-hammer.gz
772026-11-05T16:55:29.085Z+ b=crucible-hammer.gz
782026-11-05T16:55:29.085Z+ b=crucible-hammer
792026-11-05T16:55:29.085Z+ gunzip
802026-11-05T16:55:30.797Z+ chmod +x /var/tmp/bins/crucible-hammer
812026-11-05T16:55:30.800Z+ for t in "$input/bins/"*.gz
822026-11-05T16:55:30.800Z++ basename /input/build/work/bins/crutest.gz
832026-11-05T16:55:30.800Z+ b=crutest.gz
842026-11-05T16:55:30.800Z+ b=crutest
852026-11-05T16:55:30.800Z+ gunzip
862026-11-05T16:55:32.591Z+ chmod +x /var/tmp/bins/crutest
872026-11-05T16:55:32.594Z+ for t in "$input/bins/"*.gz
882026-11-05T16:55:32.594Z++ basename /input/build/work/bins/dsc.gz
892026-11-05T16:55:32.594Z+ b=dsc.gz
902026-11-05T16:55:32.594Z+ b=dsc
912026-11-05T16:55:32.594Z+ gunzip
922026-11-05T16:55:33.390Z+ chmod +x /var/tmp/bins/dsc
932026-11-05T16:55:33.393Z+ export BINDIR=/var/tmp/bins
942026-11-05T16:55:33.393Z+ BINDIR=/var/tmp/bins
952026-11-05T16:55:33.393Z+ jobpid=1094
962026-11-05T16:55:33.393Z+ echo 'Setup debug logging'
972026-11-05T16:55:33.393ZSetup debug logging
982026-11-05T16:55:33.393Z+ mkdir /tmp/debug
992026-11-05T16:55:33.393Z+ sleep 3600
1002026-11-05T16:55:33.393Z+ psrinfo -v
1012026-11-05T16:55:33.397Z+ df -h
1022026-11-05T16:55:33.401Z+ prstat -d d -mLc 1
1032026-11-05T16:55:33.404Z+ iostat -T d -xn 1
1042026-11-05T16:55:33.404Z+ mpstat -T d 1
1052026-11-05T16:55:33.404Z+ vmstat -T d -p 1
1062026-11-05T16:55:33.404Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1072026-11-05T16:55:33.406Z+ banner test_up_2r_encrypted
1082026-11-05T16:55:33.406Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1092026-11-05T16:55:33.409Z #####
1102026-11-05T16:55:33.409Z ##### ###### #### ##### # # ##### # # #####
1112026-11-05T16:55:33.409Z # # # # # # # # # # #
1122026-11-05T16:55:33.409Z # ##### #### # # # # # ##### # #
1132026-11-05T16:55:33.410Z # # # # # # ##### # #####
1142026-11-05T16:55:33.410Z # # # # # # # # # # #
1152026-11-05T16:55:33.410Z # ###### #### # ####### #### # ####### ####### # #
1162026-11-05T16:55:33.410Z
1172026-11-05T16:55:33.410Z+ ptime -m bash /input/build/work/scripts/test_up.sh -r 2 -N encrypted
1182026-11-05T16:55:33.413Z/input/build/work
1192026-11-05T16:55:33.421ZUsing 2 region sets
1202026-11-05T16:55:33.421ZTurn off color for downstairs dump
1212026-11-05T16:55:33.434ZUpstairs using key: FJ0pcoyXvopyZDieE7R1hxDAT50SNeM7cSkr7slDdS4=
1222026-11-05T16:55:33.437Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1232026-11-05T16:55:33.437ZCreating 6 downstairs regions
1242026-11-05T16:55:33.696ZStarting 6 downstairs
1252026-11-05T16:55:33.700Zdsc started at PID: 1142
1262026-11-05T16:55:38.784ZDisable automatic restart on all downstairs
1272026-11-05T16:55:38.797Z
1282026-11-05T16:55:38.797ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1292026-11-05T16:55:38.801ZRunning test: span
1302026-11-05T16:55:42.873ZCompleted test: span
1312026-11-05T16:55:47.880ZRunning test: big
1322026-11-05T16:55:52.173ZCompleted test: big
1332026-11-05T16:55:57.179ZRunning test: dep
1342026-11-05T16:56:16.172ZCompleted test: dep
1352026-11-05T16:56:21.179ZRunning test: balloon
1362026-11-05T16:58:41.389ZCompleted test: balloon
1372026-11-05T16:58:46.396ZRunning test: deactivate
1382026-11-05T16:58:50.751ZCompleted test: deactivate
1392026-11-05T16:58:55.758ZRunning hammer
1402026-11-05T16:59:07.551ZRun repair tests
1412026-11-05T16:59:07.554Z/var/tmp/bins/crutest fill -g 31 -q --key FJ0pcoyXvopyZDieE7R1hxDAT50SNeM7cSkr7slDdS4= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1422026-11-05T16:59:07.572Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.617979769Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
1432026-11-05T16:59:07.577Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.618351839Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
1442026-11-05T16:59:07.577Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.61910688Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
1452026-11-05T16:59:07.577Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.61937252Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
1462026-11-05T16:59:07.577Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f3cc313453d17841dfe67f7fc42817b041a2fc86\",\n git_commit_timestamp: \"2026-11-05T16:40:42.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.61940353Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
1472026-11-05T16:59:07.577Z{"msg":"Upstairs <-> Downstairs Message Version: 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.61941741Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
1482026-11-05T16:59:07.577Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.61942998Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
1492026-11-05T16:59:07.577Z{"msg":"Crucible 20d276b1-c59e-4093-9455-44609608c932 has session id: 9928bbef-e87a-4841-af94-78c332afb1b4","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.619909001Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1502026-11-05T16:59:07.577Z{"msg":"Upstairs opts: Upstairs UUID: 20d276b1-c59e-4093-9455-44609608c932, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.619950021Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1512026-11-05T16:59:07.577Z{"msg":"Crucible stats registered with UUID: 20d276b1-c59e-4093-9455-44609608c932","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.619967441Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1522026-11-05T16:59:07.577Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:06.619978751Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1532026-11-05T16:59:07.577Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.620121701Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1542026-11-05T16:59:07.577Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.620172051Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1552026-11-05T16:59:07.577Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.620186901Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1562026-11-05T16:59:07.577Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621045582Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
1572026-11-05T16:59:07.577Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621125082Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
1582026-11-05T16:59:07.577Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f3cc313453d17841dfe67f7fc42817b041a2fc86\",\n git_commit_timestamp: \"2026-11-05T16:40:42.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621145392Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
1592026-11-05T16:59:07.577Z{"msg":"Upstairs <-> Downstairs Message Version: 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621155122Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
1602026-11-05T16:59:07.577Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621167252Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
1612026-11-05T16:59:07.577Z{"msg":"Crucible 20d276b1-c59e-4093-9455-44609608c932 has session id: 33f0c95a-5014-4c88-8f3b-be2c052ea0bd","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621416303Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
1622026-11-05T16:59:07.578Z{"msg":"Upstairs opts: Upstairs UUID: 20d276b1-c59e-4093-9455-44609608c932, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621439443Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
1632026-11-05T16:59:07.578Z{"msg":"Crucible stats registered with UUID: 20d276b1-c59e-4093-9455-44609608c932","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621450643Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
1642026-11-05T16:59:07.578Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:06.621461033Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
1652026-11-05T16:59:07.578Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621478583Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
1662026-11-05T16:59:07.578Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621537553Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
1672026-11-05T16:59:07.578Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621553013Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
1682026-11-05T16:59:07.578Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621694453Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
1692026-11-05T16:59:07.578Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621707073Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
1702026-11-05T16:59:07.578Z{"msg":"20d276b1-c59e-4093-9455-44609608c932 active request set","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621742973Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1712026-11-05T16:59:07.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621753693Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1722026-11-05T16:59:07.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621765183Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1732026-11-05T16:59:07.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621776133Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1742026-11-05T16:59:07.578Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621786973Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1752026-11-05T16:59:07.578Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621797563Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1762026-11-05T16:59:07.578Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621808103Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1772026-11-05T16:59:07.583Z{"msg":"ds_connection connected from Ok(127.0.0.1:48399)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621858163Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1782026-11-05T16:59:07.583Z{"msg":"ds_connection connected from Ok(127.0.0.1:61868)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621877953Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1792026-11-05T16:59:07.583Z{"msg":"ds_connection connected from Ok(127.0.0.1:37295)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.621889213Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1802026-11-05T16:59:07.583Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.622348714Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1812026-11-05T16:59:07.583Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.622375814Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1822026-11-05T16:59:07.583Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.622415244Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1832026-11-05T16:59:07.583Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.622625274Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1842026-11-05T16:59:07.583Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.622653984Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1852026-11-05T16:59:07.583Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.622666334Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1862026-11-05T16:59:07.583Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.622749074Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1872026-11-05T16:59:07.584Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.622764744Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1882026-11-05T16:59:07.584Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.622775244Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1892026-11-05T16:59:07.584Z{"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.622993404Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1902026-11-05T16:59:07.584Z{"msg":"[0]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623011184Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1912026-11-05T16:59:07.584Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623022454Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1922026-11-05T16:59:07.584Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623037184Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1932026-11-05T16:59:07.584Z{"msg":"[1]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623047974Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1942026-11-05T16:59:07.584Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623062834Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1952026-11-05T16:59:07.584Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623073424Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1962026-11-05T16:59:07.584Z{"msg":"[2]R flush_numbers: [3997, 3997, 3997, 3997, 3997]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623088344Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1972026-11-05T16:59:07.584Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623098644Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1982026-11-05T16:59:07.584Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623109334Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
1992026-11-05T16:59:07.584Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623119734Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2002026-11-05T16:59:07.584Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623130295Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2012026-11-05T16:59:07.584Z{"msg":"Next flush: 3998","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623144865Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2022026-11-05T16:59:07.584Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623155155Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2032026-11-05T16:59:07.584Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623169895Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2042026-11-05T16:59:07.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623179845Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2052026-11-05T16:59:07.584Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623190115Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2062026-11-05T16:59:07.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623200715Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2072026-11-05T16:59:07.584Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623214855Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2082026-11-05T16:59:07.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623229255Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2092026-11-05T16:59:07.584Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623239185Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2102026-11-05T16:59:07.584Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623253715Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2112026-11-05T16:59:07.584Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623263285Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2122026-11-05T16:59:07.584Z{"msg":"20d276b1-c59e-4093-9455-44609608c932 is now active with session: 9928bbef-e87a-4841-af94-78c332afb1b4","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623278225Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2132026-11-05T16:59:07.584Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623288225Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
2142026-11-05T16:59:07.584Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623298015Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
2152026-11-05T16:59:07.584Z{"msg":"Activated sub_volume 20d276b1-c59e-4093-9455-44609608c932","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623311025Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
2162026-11-05T16:59:07.584Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623319755Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
2172026-11-05T16:59:07.584Z{"msg":"20d276b1-c59e-4093-9455-44609608c932 active request set","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623328685Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2182026-11-05T16:59:07.584Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623338055Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2192026-11-05T16:59:07.584Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623348745Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2202026-11-05T16:59:07.584Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623381705Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2212026-11-05T16:59:07.584Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623402435Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2222026-11-05T16:59:07.584Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623468115Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2232026-11-05T16:59:07.584Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623498845Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2242026-11-05T16:59:07.584Z{"msg":"ds_connection connected from Ok(127.0.0.1:41000)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623510985Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2252026-11-05T16:59:07.585Z{"msg":"ds_connection connected from Ok(127.0.0.1:50346)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623526755Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2262026-11-05T16:59:07.585Z{"msg":"ds_connection connected from Ok(127.0.0.1:34793)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623538315Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2272026-11-05T16:59:07.585Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623553205Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2282026-11-05T16:59:07.585Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623630235Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2292026-11-05T16:59:07.585Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623645525Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2302026-11-05T16:59:07.585Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623777635Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2312026-11-05T16:59:07.585Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623797825Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2322026-11-05T16:59:07.585Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623898485Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2332026-11-05T16:59:07.585Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623914015Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2342026-11-05T16:59:07.585Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.623925655Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2352026-11-05T16:59:07.585Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624000155Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2362026-11-05T16:59:07.585Z{"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624161006Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2372026-11-05T16:59:07.585Z{"msg":"[0]R flush_numbers: [5321, 5319, 5321, 5318, 5311]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624175426Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2382026-11-05T16:59:07.585Z{"msg":"[0]R generation: [14, 14, 14, 13, 11]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624195856Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2392026-11-05T16:59:07.585Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624210446Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2402026-11-05T16:59:07.585Z{"msg":"[1]R flush_numbers: [5321, 5319, 5321, 5318, 5311]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624221076Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2412026-11-05T16:59:07.585Z{"msg":"[1]R generation: [14, 14, 14, 13, 11]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624239246Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2422026-11-05T16:59:07.585Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624253906Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2432026-11-05T16:59:07.585Z{"msg":"[2]R flush_numbers: [5321, 5319, 5321, 5318, 5311]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624264406Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2442026-11-05T16:59:07.585Z{"msg":"[2]R generation: [14, 14, 14, 13, 11]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624279366Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2452026-11-05T16:59:07.585Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624289776Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2462026-11-05T16:59:07.585Z{"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624304126Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2472026-11-05T16:59:07.585Z{"msg":"Generation requested: 31 >= found:15","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624314656Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2482026-11-05T16:59:07.585Z{"msg":"Next flush: 5322","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624328696Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2492026-11-05T16:59:07.585Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624338906Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"downstairs","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2502026-11-05T16:59:07.585Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624353766Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2512026-11-05T16:59:07.585Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624363486Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2522026-11-05T16:59:07.585Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624381216Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"0","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2532026-11-05T16:59:07.585Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624392396Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2542026-11-05T16:59:07.585Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624412276Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"1","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2552026-11-05T16:59:07.585Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624423246Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2562026-11-05T16:59:07.585Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624434036Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2572026-11-05T16:59:07.585Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624445106Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2582026-11-05T16:59:07.585Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624455566Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2592026-11-05T16:59:07.585Z{"msg":"20d276b1-c59e-4093-9455-44609608c932 is now active with session: 33f0c95a-5014-4c88-8f3b-be2c052ea0bd","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624466616Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2602026-11-05T16:59:07.585Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624476376Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
2612026-11-05T16:59:07.585Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624486486Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
2622026-11-05T16:59:07.585Z{"msg":"Activated sub_volume 20d276b1-c59e-4093-9455-44609608c932","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624495636Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
2632026-11-05T16:59:07.585Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624504186Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
2642026-11-05T16:59:07.586ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
2652026-11-05T16:59:07.586Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.624560896Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
2662026-11-05T16:59:07.602ZRead and Verify all blocks (0..100 range:false)
2672026-11-05T16:59:07.648Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.693669885Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","----------------------------------------------------------------
2682026-11-05T16:59:07.648Zpid": Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2692026-11-05T16:59:07.648Z1185States: Active," Activetask" Active:"
2702026-11-05T16:59:07.648Zcrutest"}
2712026-11-05T16:59:07.648Z{JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2722026-11-05T16:59:07.648Z"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30 1002 Acked Read 50," Donetime Done": Done" false
2732026-11-05T16:59:07.648Z2026-11-05T16:59:06.693706145Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
2742026-11-05T16:59:07.652Z STATES DS:0 DS:1 DS:2 TOTAL
2752026-11-05T16:59:07.652Z Sent 0 0 0 0
2762026-11-05T16:59:07.652Z Done 1 1 1 3
2772026-11-05T16:59:07.652Z Skipped 0 0 0 0
2782026-11-05T16:59:07.652Z Error 0 0 0 0
2792026-11-05T16:59:07.652ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2802026-11-05T16:59:07.652ZDownstairs last five completed: 1001 1000
2812026-11-05T16:59:07.652ZUpstairs last five completed: 1002 1001 1000
2822026-11-05T16:59:07.652Z----------------------------------------------------------------
2832026-11-05T16:59:07.652Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2842026-11-05T16:59:07.652ZStates: Active Active Active
2852026-11-05T16:59:07.652ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2862026-11-05T16:59:07.652Z 1002 Acked Read 50 Done Done Done false
2872026-11-05T16:59:07.652Z STATES DS:0 DS:1 DS:2 TOTAL
2882026-11-05T16:59:07.652Z Sent 0 0 0 0
2892026-11-05T16:59:07.652Z Done 1 1 1 3
2902026-11-05T16:59:07.652Z Skipped 0 0 0 0
2912026-11-05T16:59:07.652Z Error 0 0 0 0
2922026-11-05T16:59:07.652ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
2932026-11-05T16:59:07.652ZDownstairs last five completed: 1001 1000
2942026-11-05T16:59:07.652ZUpstairs last five completed: 1002 1001 1000
2952026-11-05T16:59:07.652Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:06.693803945Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
2962026-11-05T16:59:11.649Z----------------------------------------------------------------
2972026-11-05T16:59:11.649Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2982026-11-05T16:59:11.649ZDownstairs last five completed: 1003 1002 1001 1000
2992026-11-05T16:59:11.649ZUpstairs last five completed: 1003 1002 1001 1000
3002026-11-05T16:59:11.653Z----------------------------------------------------------------
3012026-11-05T16:59:11.653Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
3022026-11-05T16:59:11.653ZDownstairs last five completed: 1003 1002 1001 1000
3032026-11-05T16:59:11.653ZUpstairs last five completed: 1003 1002 1001 1000
3042026-11-05T16:59:11.653Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.695429531Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
3052026-11-05T16:59:11.653Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.695500001Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"task":"crutest"}
3062026-11-05T16:59:11.653Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:10.695518121Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
3072026-11-05T16:59:11.653Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:10.695539481Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185}
3082026-11-05T16:59:11.653Z{"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:10.695557221Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
3092026-11-05T16:59:11.653Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:10.695582501Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
3102026-11-05T16:59:11.654Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:10.695604821Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
3112026-11-05T16:59:11.654Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.695626881Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"9928bbef-e87a-4841-af94-78c332afb1b4"}
3122026-11-05T16:59:11.654Z{"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.695652651Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
3132026-11-05T16:59:11.654Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:10.695737161Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
3142026-11-05T16:59:11.654Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:10.695759451Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
3152026-11-05T16:59:11.654Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.695781091Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"33f0c95a-5014-4c88-8f3b-be2c052ea0bd"}
3162026-11-05T16:59:11.659ZRepair setup passed
3172026-11-05T16:59:11.662ZCopy the region for /var/tmp/test_up-build/8830
3182026-11-05T16:59:11.662Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
3192026-11-05T16:59:11.666Z/var/tmp/bins/crutest fill -g 32 -q --key FJ0pcoyXvopyZDieE7R1hxDAT50SNeM7cSkr7slDdS4= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3202026-11-05T16:59:11.683Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.728475689Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
3212026-11-05T16:59:11.686Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.728876609Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
3222026-11-05T16:59:11.686Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.72969835Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
3232026-11-05T16:59:11.686Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.72994657Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3242026-11-05T16:59:11.686Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f3cc313453d17841dfe67f7fc42817b041a2fc86\",\n git_commit_timestamp: \"2026-11-05T16:40:42.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.72997486Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3252026-11-05T16:59:11.686Z{"msg":"Upstairs <-> Downstairs Message Version: 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.72998768Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3262026-11-05T16:59:11.686Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.72999999Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3272026-11-05T16:59:11.686Z{"msg":"Crucible e291a01a-67a5-4e47-9b12-a029eed5170c has session id: 0770afc5-405e-49b8-ba4a-be58bc9801be","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.730444931Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3282026-11-05T16:59:11.686Z{"msg":"Upstairs opts: Upstairs UUID: e291a01a-67a5-4e47-9b12-a029eed5170c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.730468871Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3292026-11-05T16:59:11.686Z{"msg":"Crucible stats registered with UUID: e291a01a-67a5-4e47-9b12-a029eed5170c","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.730479431Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3302026-11-05T16:59:11.686Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:10.730492261Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3312026-11-05T16:59:11.686Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.730661851Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3322026-11-05T16:59:11.686Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.730689271Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3332026-11-05T16:59:11.686Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.730700191Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3342026-11-05T16:59:11.686Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.731928403Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
3352026-11-05T16:59:11.686Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732001743Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3362026-11-05T16:59:11.687Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f3cc313453d17841dfe67f7fc42817b041a2fc86\",\n git_commit_timestamp: \"2026-11-05T16:40:42.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732025903Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3372026-11-05T16:59:11.687Z{"msg":"Upstairs <-> Downstairs Message Version: 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732038783Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3382026-11-05T16:59:11.687Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732050773Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3392026-11-05T16:59:11.691Z{"msg":"Crucible e291a01a-67a5-4e47-9b12-a029eed5170c has session id: f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732367963Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3402026-11-05T16:59:11.692Z{"msg":"Upstairs opts: Upstairs UUID: e291a01a-67a5-4e47-9b12-a029eed5170c, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732395333Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3412026-11-05T16:59:11.692Z{"msg":"Crucible stats registered with UUID: e291a01a-67a5-4e47-9b12-a029eed5170c","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732405853Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3422026-11-05T16:59:11.692Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:10.732420043Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3432026-11-05T16:59:11.692Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732429863Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3442026-11-05T16:59:11.692Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732445093Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3452026-11-05T16:59:11.692Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732465533Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3462026-11-05T16:59:11.692Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732619683Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
3472026-11-05T16:59:11.692Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732634223Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3482026-11-05T16:59:11.692Z{"msg":"e291a01a-67a5-4e47-9b12-a029eed5170c active request set","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732647764Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3492026-11-05T16:59:11.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732661574Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3502026-11-05T16:59:11.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732672044Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3512026-11-05T16:59:11.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732706374Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3522026-11-05T16:59:11.692Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732731074Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3532026-11-05T16:59:11.692Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732742184Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3542026-11-05T16:59:11.692Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732753044Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3552026-11-05T16:59:11.692Z{"msg":"ds_connection connected from Ok(127.0.0.1:57821)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732764984Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3562026-11-05T16:59:11.692Z{"msg":"ds_connection connected from Ok(127.0.0.1:62883)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732780494Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3572026-11-05T16:59:11.692Z{"msg":"ds_connection connected from Ok(127.0.0.1:45763)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.732795704Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3582026-11-05T16:59:11.692Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733232964Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3592026-11-05T16:59:11.692Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733267634Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3602026-11-05T16:59:11.692Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733282994Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3612026-11-05T16:59:11.692Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733516165Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3622026-11-05T16:59:11.692Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733544245Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3632026-11-05T16:59:11.692Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733556275Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3642026-11-05T16:59:11.692Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733601485Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3652026-11-05T16:59:11.692Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733627855Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3662026-11-05T16:59:11.692Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733671885Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3672026-11-05T16:59:11.692Z{"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733846435Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3682026-11-05T16:59:11.692Z{"msg":"[0]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733867925Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3692026-11-05T16:59:11.692Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733879305Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3702026-11-05T16:59:11.692Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733890945Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3712026-11-05T16:59:11.692Z{"msg":"[1]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733901495Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3722026-11-05T16:59:11.692Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733912465Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3732026-11-05T16:59:11.692Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733926975Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3742026-11-05T16:59:11.692Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733941875Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3752026-11-05T16:59:11.692Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733952615Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3762026-11-05T16:59:11.692Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733967425Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3772026-11-05T16:59:11.693Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733977705Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3782026-11-05T16:59:11.693Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733988715Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3792026-11-05T16:59:11.693Z{"msg":"Next flush: 3999","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.733998645Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3802026-11-05T16:59:11.693Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734016805Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3812026-11-05T16:59:11.693Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734027385Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3822026-11-05T16:59:11.693Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734041365Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3832026-11-05T16:59:11.693Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734055995Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3842026-11-05T16:59:11.693Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734066465Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3852026-11-05T16:59:11.693Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734076445Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3862026-11-05T16:59:11.693Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734086765Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3872026-11-05T16:59:11.693Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734100905Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3882026-11-05T16:59:11.693Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734111555Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3892026-11-05T16:59:11.693Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734125625Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3902026-11-05T16:59:11.693Z{"msg":"e291a01a-67a5-4e47-9b12-a029eed5170c is now active with session: 0770afc5-405e-49b8-ba4a-be58bc9801be","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734136045Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3912026-11-05T16:59:11.693Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734149835Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"0770afc5-405e-49b8-ba4a-be58bc9801be"}
3922026-11-05T16:59:11.693Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734159565Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3932026-11-05T16:59:11.693Z{"msg":"Activated sub_volume e291a01a-67a5-4e47-9b12-a029eed5170c","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734172275Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3942026-11-05T16:59:11.693Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734180865Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
3952026-11-05T16:59:11.693Z{"msg":"e291a01a-67a5-4e47-9b12-a029eed5170c active request set","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734193925Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3962026-11-05T16:59:11.693Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734203655Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3972026-11-05T16:59:11.693Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734218025Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3982026-11-05T16:59:11.693Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734232665Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
3992026-11-05T16:59:11.693Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734242845Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4002026-11-05T16:59:11.693Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734254145Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4012026-11-05T16:59:11.693Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734265165Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4022026-11-05T16:59:11.693Z{"msg":"ds_connection connected from Ok(127.0.0.1:61354)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734276055Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4032026-11-05T16:59:11.693Z{"msg":"ds_connection connected from Ok(127.0.0.1:53485)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734290725Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4042026-11-05T16:59:11.693Z{"msg":"ds_connection connected from Ok(127.0.0.1:50624)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734388895Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4052026-11-05T16:59:11.693Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734456056Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4062026-11-05T16:59:11.693Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734492696Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4072026-11-05T16:59:11.693Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734513996Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4082026-11-05T16:59:11.693Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734713526Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4092026-11-05T16:59:11.693Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734751096Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4102026-11-05T16:59:11.693Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734767936Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4112026-11-05T16:59:11.693Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734806026Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4122026-11-05T16:59:11.693Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734844606Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4132026-11-05T16:59:11.693Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.734856016Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4142026-11-05T16:59:11.693Z{"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735019146Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4152026-11-05T16:59:11.694Z{"msg":"[0]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735030256Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4162026-11-05T16:59:11.694Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735041436Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4172026-11-05T16:59:11.694Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735052716Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4182026-11-05T16:59:11.694Z{"msg":"[1]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735063576Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4192026-11-05T16:59:11.694Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735074426Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4202026-11-05T16:59:11.694Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735084976Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4212026-11-05T16:59:11.694Z{"msg":"[2]R flush_numbers: [5322, 5322, 5322, 5322, 5322]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735095526Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4222026-11-05T16:59:11.694Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735106096Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4232026-11-05T16:59:11.694Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735116796Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4242026-11-05T16:59:11.694Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735126786Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4252026-11-05T16:59:11.694Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735141146Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4262026-11-05T16:59:11.694Z{"msg":"Next flush: 5323","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735151206Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4272026-11-05T16:59:11.694Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735165416Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"":"downstairs","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4282026-11-05T16:59:11.694Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735175576Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4292026-11-05T16:59:11.694Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735189036Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4302026-11-05T16:59:11.694Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735198946Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"0","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4312026-11-05T16:59:11.694Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735213236Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4322026-11-05T16:59:11.694Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735223036Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"1","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4332026-11-05T16:59:11.694Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735237666Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4342026-11-05T16:59:11.694Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735247646Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"client":"2","":"downstairs","session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4352026-11-05T16:59:11.694Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735261916Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4362026-11-05T16:59:11.694Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735271917Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4372026-11-05T16:59:11.694Z{"msg":"e291a01a-67a5-4e47-9b12-a029eed5170c is now active with session: f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735282567Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4382026-11-05T16:59:11.694Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735296437Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"session_id":"f8f95d96-c4a1-48c7-a38d-d4f7df8b3f19"}
4392026-11-05T16:59:11.699Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735306147Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
4402026-11-05T16:59:11.699Z{"msg":"Activated sub_volume e291a01a-67a5-4e47-9b12-a029eed5170c","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735318867Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
4412026-11-05T16:59:11.699Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735327267Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
4422026-11-05T16:59:11.699ZDisk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
4432026-11-05T16:59:11.699ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4442026-11-05T16:59:11.699Z{"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.735769037Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
4452026-11-05T16:59:11.713ZRead and Verify all blocks (0..100 range:false)
4462026-11-05T16:59:11.759Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.804490836Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
4472026-11-05T16:59:11.759Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.804519956Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"----------------------------------------------------------------
4482026-11-05T16:59:11.759Z}
4492026-11-05T16:59:11.759Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4502026-11-05T16:59:11.759ZStates: Active Active Active
4512026-11-05T16:59:11.762ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4522026-11-05T16:59:11.762Z 1002 Acked Read 50 Done Done Done false
4532026-11-05T16:59:11.762Z STATES DS:0 DS:1 DS:2 TOTAL
4542026-11-05T16:59:11.762Z Sent 0 0 0 0
4552026-11-05T16:59:11.762Z Done 1 1 1 3
4562026-11-05T16:59:11.762Z Skipped 0 0 0 0
4572026-11-05T16:59:11.763Z Error 0 0 0 0
4582026-11-05T16:59:11.763ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4592026-11-05T16:59:11.763ZDownstairs last five completed: 1001 1000
4602026-11-05T16:59:11.763ZUpstairs last five completed: 1002 1001 1000
4612026-11-05T16:59:11.763Z----------------------------------------------------------------
4622026-11-05T16:59:11.763Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
4632026-11-05T16:59:11.763ZStates: Active Active Active
4642026-11-05T16:59:11.763ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
4652026-11-05T16:59:11.763Z 1002 Acked Read 50 Done Done Done false
4662026-11-05T16:59:11.763Z STATES DS:0 DS:1 DS:2 TOTAL
4672026-11-05T16:59:11.763Z Sent 0 0 0 0
4682026-11-05T16:59:11.763Z Done 1 1 1 3
4692026-11-05T16:59:11.763Z Skipped 0 0 0 0
4702026-11-05T16:59:11.763Z Error 0 0 0 0
4712026-11-05T16:59:11.763ZLast Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001))
4722026-11-05T16:59:11.763ZDownstairs last five completed: 1001 1000
4732026-11-05T16:59:11.763ZUpstairs last five completed: 1002 1001 1000
4742026-11-05T16:59:11.763Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:10.804632766Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
4752026-11-05T16:59:15.760Z----------------------------------------------------------------
4762026-11-05T16:59:15.760Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4772026-11-05T16:59:15.760ZDownstairs last five completed: 1003 1002 1001 1000
4782026-11-05T16:59:15.760ZUpstairs last five completed: 1003 1002 1001 1000
4792026-11-05T16:59:15.760Z----------------------------------------------------------------
4802026-11-05T16:59:15.760Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
4812026-11-05T16:59:15.760ZDownstairs last five completed: 1003 1002 1001 1000
4822026-11-05T16:59:15.760ZUpstairs last five completed: 1003 1002 1001 1000
4832026-11-05T16:59:15.764Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:14.805598901Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
4842026-11-05T16:59:15.764Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:14.805675711Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191,"task":"crutest"}
4852026-11-05T16:59:15.764Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:14.805699371Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1191}
4862026-11-05T16:59:15.767ZRepair part 1 passed
4872026-11-05T16:59:15.767Z
4882026-11-05T16:59:15.767ZKill the current downstairs
4892026-11-05T16:59:15.809ZDownstairs 2 stopped
4902026-11-05T16:59:15.809Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
4912026-11-05T16:59:15.812ZNow put back the original so we have a mismatch
4922026-11-05T16:59:15.812Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
4932026-11-05T16:59:15.812ZRestart downstairs with old directory
4942026-11-05T16:59:15.837Zdownstairs 2 not restarted yet, waiting
4952026-11-05T16:59:20.856ZDownstairs 2 restarted
4962026-11-05T16:59:20.860Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
4972026-11-05T16:59:20.869ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4982026-11-05T16:59:20.873Z 0 000-009 32 32 31 3999 3999 3998 F F F <---
4992026-11-05T16:59:20.873Z 1 010-019 32 32 31 3999 3999 3998 F F F <---
5002026-11-05T16:59:20.873Z 2 020-029 32 32 31 3999 3999 3998 F F F <---
5012026-11-05T16:59:20.873Z 3 030-039 32 32 31 3999 3999 3998 F F F <---
5022026-11-05T16:59:20.873Z 4 040-049 32 32 31 3999 3999 3998 F F F <---
5032026-11-05T16:59:20.873ZMax gen: 32, Max flush: 3999
5042026-11-05T16:59:20.873ZError: Difference in extent metadata found!
5052026-11-05T16:59:20.873Zdump test found error as expected
5062026-11-05T16:59:20.873Z
5072026-11-05T16:59:20.873Z
5082026-11-05T16:59:20.873Z/var/tmp/bins/crutest verify --range -g 33 -q --key FJ0pcoyXvopyZDieE7R1hxDAT50SNeM7cSkr7slDdS4= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
5092026-11-05T16:59:20.890Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.936119084Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
5102026-11-05T16:59:20.895Z{"msg":"dsc has 6 regions. This means 2 sub_volumes","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.936377865Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
5112026-11-05T16:59:20.895Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.937114685Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
5122026-11-05T16:59:20.895Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.937365176Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
5132026-11-05T16:59:20.895Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f3cc313453d17841dfe67f7fc42817b041a2fc86\",\n git_commit_timestamp: \"2026-11-05T16:40:42.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.937395156Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
5142026-11-05T16:59:20.895Z{"msg":"Upstairs <-> Downstairs Message Version: 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.937408996Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
5152026-11-05T16:59:20.895Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.937421376Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
5162026-11-05T16:59:20.895Z{"msg":"Crucible 6404620f-efd0-481d-b1db-b2ff87599937 has session id: 4b98775f-a78a-4e92-8545-f5b265b3e979","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.937852596Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5172026-11-05T16:59:20.895Z{"msg":"Upstairs opts: Upstairs UUID: 6404620f-efd0-481d-b1db-b2ff87599937, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.937873576Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5182026-11-05T16:59:20.895Z{"msg":"Crucible stats registered with UUID: 6404620f-efd0-481d-b1db-b2ff87599937","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.937884986Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5192026-11-05T16:59:20.895Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:19.937895796Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5202026-11-05T16:59:20.895Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.938071327Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5212026-11-05T16:59:20.895Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.938129617Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5222026-11-05T16:59:20.895Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.938159087Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5232026-11-05T16:59:20.895Z{"msg":"SV 1 has targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939111998Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
5242026-11-05T16:59:20.895Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939171308Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
5252026-11-05T16:59:20.896Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f3cc313453d17841dfe67f7fc42817b041a2fc86\",\n git_commit_timestamp: \"2026-11-05T16:40:42.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.99.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"58cf8f9c35b355199727627d07ae34a4e2846d67\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939192688Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
5262026-11-05T16:59:20.896Z{"msg":"Upstairs <-> Downstairs Message Version: 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939202128Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
5272026-11-05T16:59:20.896Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939214788Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
5282026-11-05T16:59:20.896Z{"msg":"Crucible 6404620f-efd0-481d-b1db-b2ff87599937 has session id: 1ca60f5e-1d29-4763-86de-f0b5991a2bb2","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939493058Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
5292026-11-05T16:59:20.896Z{"msg":"Upstairs opts: Upstairs UUID: 6404620f-efd0-481d-b1db-b2ff87599937, Targets: [127.0.0.1:8840, 127.0.0.1:8850, 127.0.0.1:8860], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939531988Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
5302026-11-05T16:59:20.896Z{"msg":"Crucible stats registered with UUID: 6404620f-efd0-481d-b1db-b2ff87599937","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939544118Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
5312026-11-05T16:59:20.896Z{"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:19.939561828Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
5322026-11-05T16:59:20.896Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939571338Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
5332026-11-05T16:59:20.896Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939583578Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
5342026-11-05T16:59:20.896Z{"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939595278Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
5352026-11-05T16:59:20.896Z{"msg":"Downstairs in volume = 6","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939735308Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
5362026-11-05T16:59:20.896Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939760698Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
5372026-11-05T16:59:20.896Z{"msg":"6404620f-efd0-481d-b1db-b2ff87599937 active request set","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939802439Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5382026-11-05T16:59:20.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939817219Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5392026-11-05T16:59:20.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939827979Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5402026-11-05T16:59:20.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939861569Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5412026-11-05T16:59:20.896Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939891479Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5422026-11-05T16:59:20.896Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939903829Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5432026-11-05T16:59:20.896Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939914719Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5442026-11-05T16:59:20.896Z{"msg":"ds_connection connected from Ok(127.0.0.1:48886)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939934539Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5452026-11-05T16:59:20.896Z{"msg":"ds_connection connected from Ok(127.0.0.1:44366)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939971129Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5462026-11-05T16:59:20.896Z{"msg":"ds_connection connected from Ok(127.0.0.1:48441)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.939982209Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5472026-11-05T16:59:20.900Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.940455999Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5482026-11-05T16:59:20.901Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.940486799Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5492026-11-05T16:59:20.901Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.940566839Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5502026-11-05T16:59:20.901Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94080466Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5512026-11-05T16:59:20.901Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94084738Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5522026-11-05T16:59:20.901Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94087164Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5532026-11-05T16:59:20.901Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94089329Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5542026-11-05T16:59:20.901Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94090803Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5552026-11-05T16:59:20.901Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94098751Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5562026-11-05T16:59:20.901Z{"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94117102Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5572026-11-05T16:59:20.901Z{"msg":"[0]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94119322Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5582026-11-05T16:59:20.901Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.9412573Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5592026-11-05T16:59:20.901Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94128553Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5602026-11-05T16:59:20.901Z{"msg":"[1]R flush_numbers: [3999, 3999, 3999, 3999, 3999]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94130128Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5612026-11-05T16:59:20.901Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94131603Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5622026-11-05T16:59:20.901Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94133105Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5632026-11-05T16:59:20.901Z{"msg":"[2]R flush_numbers: [3998, 3998, 3998, 3998, 3998]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94134584Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5642026-11-05T16:59:20.901Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94136045Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5652026-11-05T16:59:20.901Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94137492Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5662026-11-05T16:59:20.901Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94138917Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5672026-11-05T16:59:20.901Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.9414002Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5682026-11-05T16:59:20.901Z{"msg":"Next flush: 4000","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94141443Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5692026-11-05T16:59:20.901Z{"msg":"extent 0 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.9414332Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5702026-11-05T16:59:20.901Z{"msg":"extent:0 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94145296Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5712026-11-05T16:59:20.901Z{"msg":"found dest for source 0 for extent at index 0 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94146866Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5722026-11-05T16:59:20.901Z{"msg":"extent 1 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94148712Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5732026-11-05T16:59:20.901Z{"msg":"extent:1 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94150538Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5742026-11-05T16:59:20.901Z{"msg":"found dest for source 0 for extent at index 1 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.94152089Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5752026-11-05T16:59:20.901Z{"msg":"extent 2 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941539311Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5762026-11-05T16:59:20.901Z{"msg":"extent:2 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941558211Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5772026-11-05T16:59:20.901Z{"msg":"found dest for source 0 for extent at index 2 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941573331Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5782026-11-05T16:59:20.901Z{"msg":"extent 3 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941592161Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5792026-11-05T16:59:20.901Z{"msg":"extent:3 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941610421Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5802026-11-05T16:59:20.901Z{"msg":"found dest for source 0 for extent at index 3 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941625691Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5812026-11-05T16:59:20.901Z{"msg":"extent 4 needs reconciliation: ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })]))","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941640171Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5822026-11-05T16:59:20.901Z{"msg":"extent:4 ClientMap(ClientData([Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 32, flush: 3999, dirty: false }), Some(ExtentMetadata { generation: 31, flush: 3998, dirty: false })])) => 0","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941658421Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5832026-11-05T16:59:20.902Z{"msg":"found dest for source 0 for extent at index 4 => [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941669591Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"mend","":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5842026-11-05T16:59:20.902Z{"msg":"4b98775f-a78a-4e92-8545-f5b265b3e979 [0] setting state to reconcile from Connecting { state: WaitQuorum, mode: New }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941689891Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5852026-11-05T16:59:20.902Z{"msg":"4b98775f-a78a-4e92-8545-f5b265b3e979 [1] setting state to reconcile from Connecting { state: WaitQuorum, mode: New }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941706161Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5862026-11-05T16:59:20.902Z{"msg":"4b98775f-a78a-4e92-8545-f5b265b3e979 [2] setting state to reconcile from Connecting { state: WaitQuorum, mode: New }","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941717501Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5872026-11-05T16:59:20.902Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941743021Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5882026-11-05T16:59:20.902Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 4000, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941837171Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5892026-11-05T16:59:20.902Z{"msg":"starting reconciliation af000c37-9383-4a1a-8b9e-fc9d4ed34b39: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941849341Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5902026-11-05T16:59:20.902Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.941864381Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5912026-11-05T16:59:20.907Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.953220804Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5922026-11-05T16:59:20.911Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.955244646Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5932026-11-05T16:59:20.914Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.957236768Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5942026-11-05T16:59:20.914Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.959156971Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5952026-11-05T16:59:20.919Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960801793Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5962026-11-05T16:59:20.919Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960830953Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5972026-11-05T16:59:20.919Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960850393Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5982026-11-05T16:59:20.919Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960865563Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
5992026-11-05T16:59:20.919Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960880273Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
6002026-11-05T16:59:20.919Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960891133Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
6012026-11-05T16:59:20.919Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960901533Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
6022026-11-05T16:59:20.919Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960916563Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
6032026-11-05T16:59:20.919Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960927223Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
6042026-11-05T16:59:20.919Z{"msg":"6404620f-efd0-481d-b1db-b2ff87599937 is now active with session: 4b98775f-a78a-4e92-8545-f5b265b3e979","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960942283Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"4b98775f-a78a-4e92-8545-f5b265b3e979"}
6052026-11-05T16:59:20.919Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960956353Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
6062026-11-05T16:59:20.919Z{"msg":"Activated sub_volume 6404620f-efd0-481d-b1db-b2ff87599937","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960965783Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
6072026-11-05T16:59:20.919Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960978813Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
6082026-11-05T16:59:20.919Z{"msg":"6404620f-efd0-481d-b1db-b2ff87599937 active request set","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.960987453Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6092026-11-05T16:59:20.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961001363Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6102026-11-05T16:59:20.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961011743Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6112026-11-05T16:59:20.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961026263Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6122026-11-05T16:59:20.919Z{"msg":"connecting to 127.0.0.1:8860","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961040843Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6132026-11-05T16:59:20.920Z{"msg":"connecting to 127.0.0.1:8840","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961051943Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6142026-11-05T16:59:20.920Z{"msg":"connecting to 127.0.0.1:8850","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961063493Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6152026-11-05T16:59:20.920Z{"msg":"ds_connection connected from Ok(127.0.0.1:45319)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961080473Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6162026-11-05T16:59:20.920Z{"msg":"ds_connection connected from Ok(127.0.0.1:52580)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961092063Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6172026-11-05T16:59:20.920Z{"msg":"ds_connection connected from Ok(127.0.0.1:50590)","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961195093Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6182026-11-05T16:59:20.920Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961307553Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6192026-11-05T16:59:20.920Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961374543Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6202026-11-05T16:59:20.920Z{"msg":"negotiated version 14","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961408783Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6212026-11-05T16:59:20.920Z{"msg":"downstairs client at Some(127.0.0.1:8860) has region UUID 12345678-0000-0000-0000-000000008860","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961594103Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6222026-11-05T16:59:20.920Z{"msg":"downstairs client at Some(127.0.0.1:8850) has region UUID 12345678-0000-0000-0000-000000008850","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961611023Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6232026-11-05T16:59:20.920Z{"msg":"downstairs client at Some(127.0.0.1:8840) has region UUID 12345678-0000-0000-0000-000000008840","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961627193Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6242026-11-05T16:59:20.920Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961670983Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6252026-11-05T16:59:20.920Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961689634Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6262026-11-05T16:59:20.920Z{"msg":"downstairs been_active: true","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961752664Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6272026-11-05T16:59:20.920Z{"msg":"Proceed with full quorum, cancelling min-quorum reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961900264Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6282026-11-05T16:59:20.920Z{"msg":"[0]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961929724Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6292026-11-05T16:59:20.920Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961948004Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6302026-11-05T16:59:20.920Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.961994204Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6312026-11-05T16:59:20.920Z{"msg":"[1]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962024044Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6322026-11-05T16:59:20.920Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962035634Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6332026-11-05T16:59:20.920Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962050844Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6342026-11-05T16:59:20.920Z{"msg":"[2]R flush_numbers: [5323, 5323, 5323, 5323, 5323]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962061724Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6352026-11-05T16:59:20.920Z{"msg":"[2]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962076314Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6362026-11-05T16:59:20.920Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962086744Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6372026-11-05T16:59:20.920Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962097044Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6382026-11-05T16:59:20.920Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962107484Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6392026-11-05T16:59:20.921Z{"msg":"Next flush: 5324","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962117854Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6402026-11-05T16:59:20.921Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962128174Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"":"downstairs","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6412026-11-05T16:59:20.921Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962138954Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6422026-11-05T16:59:20.921Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962148614Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6432026-11-05T16:59:20.921Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962159274Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"0","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6442026-11-05T16:59:20.921Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962169614Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6452026-11-05T16:59:20.921Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962180064Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"1","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6462026-11-05T16:59:20.921Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962190454Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6472026-11-05T16:59:20.921Z{"msg":"sending YouHaveBeenActive","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962200774Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"client":"2","":"downstairs","session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6482026-11-05T16:59:20.921Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962211264Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6492026-11-05T16:59:20.921Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962221174Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6502026-11-05T16:59:20.921Z{"msg":"6404620f-efd0-481d-b1db-b2ff87599937 is now active with session: 1ca60f5e-1d29-4763-86de-f0b5991a2bb2","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962232474Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6512026-11-05T16:59:20.921Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962242124Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"session_id":"1ca60f5e-1d29-4763-86de-f0b5991a2bb2"}
6522026-11-05T16:59:20.921Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"Disk: sv:2 bs:4096 ts:409600 tb:100 max_io:100 or 409600
6532026-11-05T16:59:20.921Z2026-11-05T16:59:19.962252454Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
6542026-11-05T16:59:20.921Z{"msg":"Activated sub_volume 6404620f-efd0-481d-b1db-b2ff87599937","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962266434Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
6552026-11-05T16:59:20.921Z{"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:19.962275464Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
6562026-11-05T16:59:20.921ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
6572026-11-05T16:59:20.921ZRead and Verify all blocks (0..100 range:true)
6582026-11-05T16:59:20.963Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:20.008439047Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
6592026-11-05T16:59:20.963Z{"msg":"Verify test completed","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:20.008459317Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
6602026-11-05T16:59:20.966Z{"msg":"Wrote out file \"/var/tmp/test_up-build/verify_file\"","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:20.008513107Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
6612026-11-05T16:59:20.966Z{"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:20.008523587Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
6622026-11-05T16:59:20.966Z----------------------------------------------------------------
6632026-11-05T16:59:20.966Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6642026-11-05T16:59:20.966ZStates: Active Active Active
6652026-11-05T16:59:20.966ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6662026-11-05T16:59:20.966Z 1000 Acked Read 50 Done Done Done false
6672026-11-05T16:59:20.966Z STATES DS:0 DS:1 DS:2 TOTAL
6682026-11-05T16:59:20.967Z Sent 0 0 0 0
6692026-11-05T16:59:20.967Z Done 1 1 1 3
6702026-11-05T16:59:20.967Z Skipped 0 0 0 0
6712026-11-05T16:59:20.967Z Error 0 0 0 0
6722026-11-05T16:59:20.967ZLast Flush: None None None
6732026-11-05T16:59:20.967ZDownstairs last five completed:
6742026-11-05T16:59:20.967ZUpstairs last five completed: 1000
6752026-11-05T16:59:20.967Z----------------------------------------------------------------
6762026-11-05T16:59:20.967Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
6772026-11-05T16:59:20.967ZStates: Active Active Active
6782026-11-05T16:59:20.967ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
6792026-11-05T16:59:20.967Z 1000 Acked Read 50 Done Done Done false
6802026-11-05T16:59:20.967Z STATES DS:0 DS:1 DS:2 TOTAL
6812026-11-05T16:59:20.967Z Sent 0 0 0 0
6822026-11-05T16:59:20.967Z Done 1 1 1 3
6832026-11-05T16:59:20.967Z Skipped 0 0 0 0
6842026-11-05T16:59:20.967Z Error 0 0 0 0
6852026-11-05T16:59:20.967ZLast Flush: None None None
6862026-11-05T16:59:20.967ZDownstairs last five completed:
6872026-11-05T16:59:20.967ZUpstairs last five completed: 1000
6882026-11-05T16:59:20.967Z{"msg":"CLIENT: Up:0 ds:2 act:6","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:20.008656247Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
6892026-11-05T16:59:24.964Z----------------------------------------------------------------
6902026-11-05T16:59:24.964Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6912026-11-05T16:59:24.964ZDownstairs last five completed: 1001 1000
6922026-11-05T16:59:24.964ZUpstairs last five completed: 1001 1000
6932026-11-05T16:59:24.964Z----------------------------------------------------------------
6942026-11-05T16:59:24.964Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
6952026-11-05T16:59:24.964ZDownstairs last five completed: 1001 1000
6962026-11-05T16:59:24.964ZUpstairs last five completed: 1001 1000
6972026-11-05T16:59:24.968Z{"msg":"CLIENT: Up:0 ds:0 act:6","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:24.009661282Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
6982026-11-05T16:59:24.968Z{"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-11-05T16:59:24.009735062Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203,"task":"crutest"}
6992026-11-05T16:59:24.968Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-11-05T16:59:24.009753272Z","hostname":"w-01M6PA3NDJFXMAXPV7J39FA428","pid":1203}
7002026-11-05T16:59:24.971ZRepair part 2 passed
7012026-11-05T16:59:24.972Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
7022026-11-05T16:59:24.985ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
7032026-11-05T16:59:24.988Z 0 000-009 32 32 32 3999 3999 3999 F F F
7042026-11-05T16:59:24.988Z 1 010-019 32 32 32 3999 3999 3999 F F F
7052026-11-05T16:59:24.988Z 2 020-029 32 32 32 3999 3999 3999 F F F
7062026-11-05T16:59:24.988Z 3 030-039 32 32 32 3999 3999 3999 F F F
7072026-11-05T16:59:24.988Z 4 040-049 32 32 32 3999 3999 3999 F F F
7082026-11-05T16:59:24.988ZMax gen: 32, Max flush: 3999
7092026-11-05T16:59:24.988Zdump test passed
7102026-11-05T16:59:24.988Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
7112026-11-05T16:59:24.999Z Extent 1
7122026-11-05T16:59:25.002ZGEN 32 32 32
7132026-11-05T16:59:25.002ZFLUSH_ID 3999 3999 3999
7142026-11-05T16:59:25.002ZDIRTY
7152026-11-05T16:59:25.002Z
7162026-11-05T16:59:25.002ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
7172026-11-05T16:59:25.002Z010 A A A A A A
7182026-11-05T16:59:25.002Z011 A A A A A A
7192026-11-05T16:59:25.002Z012 A A A A A A
7202026-11-05T16:59:25.003Z013 A A A A A A
7212026-11-05T16:59:25.003Z014 A A A A A A
7222026-11-05T16:59:25.006Z015 A A A A A A
7232026-11-05T16:59:25.006Z016 A A A A A A
7242026-11-05T16:59:25.006Z017 A A A A A A
7252026-11-05T16:59:25.006Z018 A A A A A A
7262026-11-05T16:59:25.006Z019 A A A A A A
7272026-11-05T16:59:25.009Zdump extent test passed
7282026-11-05T16:59:25.009Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
7292026-11-05T16:59:25.018ZExtent 2 Block in extent 0 Actual block 20
7302026-11-05T16:59:25.018Z
7312026-11-05T16:59:25.021Z DATA SHA256 VER
7322026-11-05T16:59:25.021Z------ ---------------------------------------------------------------- ---
7332026-11-05T16:59:25.021Z 0 16f836dc46c822358b317e8561f9eede78b233e8fad889920d4575dd6d1c1ef6 A
7342026-11-05T16:59:25.021Z 1 16f836dc46c822358b317e8561f9eede78b233e8fad889920d4575dd6d1c1ef6 A
7352026-11-05T16:59:25.021Z 2 16f836dc46c822358b317e8561f9eede78b233e8fad889920d4575dd6d1c1ef6 A
7362026-11-05T16:59:25.022Z
7372026-11-05T16:59:25.022ZNONCES 0 1 2 DIFF
7382026-11-05T16:59:25.022Z------ ------------------------ ------------------------ ------------------------ -----
7392026-11-05T16:59:25.022Z 0 3321f248f2d78ba552895952 3321f248f2d78ba552895952 3321f248f2d78ba552895952
7402026-11-05T16:59:25.022Z
7412026-11-05T16:59:25.022Z TAGS 0 1 2 DIFF
7422026-11-05T16:59:25.022Z------ -------------------------------- -------------------------------- -------------------------------- -----
7432026-11-05T16:59:25.022Z 0 75d65594f0e2b76fc759d0839717d48b 75d65594f0e2b76fc759d0839717d48b 75d65594f0e2b76fc759d0839717d48b
7442026-11-05T16:59:25.022Z
7452026-11-05T16:59:25.022ZHASHES 0 1 2 DIFF
7462026-11-05T16:59:25.022Z------ ---------------- ---------------- ---------------- -----
7472026-11-05T16:59:25.022Z
7482026-11-05T16:59:25.025Zdump block test passed
7492026-11-05T16:59:25.025ZInitial upstairs tests have completed, stopping all downstairs
7502026-11-05T16:59:30.042ZCreating 6 larger downstairs regions
7512026-11-05T16:59:30.215ZStarting 6 downstairs
7522026-11-05T16:59:35.221Zdsc restarted at PID: 1217
7532026-11-05T16:59:35.310ZNow do the two_of_three test
7542026-11-05T17:02:56.451ZCompleted test: two-of-three
7552026-11-05T17:02:56.451ZAll tests have completed, stopping all downstairs
7562026-11-05T17:02:56.467Z
7572026-11-05T17:02:56.472ZAll Tests have passed
7582026-11-05T17:02:56.472Z7:23 Test duration
7592026-11-05T17:02:56.472Z
7602026-11-05T17:02:56.472Zreal 7:23.059329671
7612026-11-05T17:02:56.472Zuser 6:41.118253149
7622026-11-05T17:02:56.472Zsys 10.503259556
7632026-11-05T17:02:56.472Ztrap 0.192109848
7642026-11-05T17:02:56.472Ztflt 0.048177674
7652026-11-05T17:02:56.472Zdflt 0.019590783
7662026-11-05T17:02:56.472Zkflt 0.000994620
7672026-11-05T17:02:56.472Zlock 13:58:09.032830953
7682026-11-05T17:02:56.472Zslp 41:08.668485425
7692026-11-05T17:02:56.472Zlat 13.398165789
7702026-11-05T17:02:56.472Zstop 0.001744570
7712026-11-05T17:02:56.472Z+ echo 'test-up-2region-encrypted ends'
7722026-11-05T17:02:56.472Ztest-up-2region-encrypted ends
7732026-11-05T17:03:01.473Zprocess exited: duration 449242 ms, exit code 0
7742026-11-05T17:03:01.473Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
7752026-11-05T17:04:01.521Zexec warning: : stdout descriptor may be held open by a background process; giving up!
7762026-11-05T17:04:01.521Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
7772026-11-05T17:04:01.547Zfound 16 output files
7782026-11-05T17:04:01.547Zuploading: /tmp/test_up-build/dsc-out.txt (48459 bytes)
7792026-11-05T17:04:02.662Zuploaded: /tmp/test_up-build/dsc-out.txt
7802026-11-05T17:04:02.662Zuploading: /tmp/test_up-build/test_up_out.txt (19323783 bytes)
7812026-11-05T17:04:03.834Zuploaded: /tmp/test_up-build/test_up_out.txt
7822026-11-05T17:04:03.834Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (108601 bytes)
7832026-11-05T17:04:04.853Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
7842026-11-05T17:04:04.853Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (1179786 bytes)
7852026-11-05T17:04:05.953Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
7862026-11-05T17:04:05.953Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (646535 bytes)
7872026-11-05T17:04:06.980Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
7882026-11-05T17:04:06.983Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (1174318 bytes)
7892026-11-05T17:04:08.032Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
7902026-11-05T17:04:08.033Zuploading: /tmp/test_up-build/dsc/downstairs-8850.txt (641033 bytes)
7912026-11-05T17:04:09.140Zuploaded: /tmp/test_up-build/dsc/downstairs-8850.txt
7922026-11-05T17:04:09.140Zuploading: /tmp/test_up-build/dsc/downstairs-8860.txt (103117 bytes)
7932026-11-05T17:04:10.418Zuploaded: /tmp/test_up-build/dsc/downstairs-8860.txt
7942026-11-05T17:04:10.419Zuploading: /tmp/debug/df.txt (1270 bytes)
7952026-11-05T17:04:11.450Zuploaded: /tmp/debug/df.txt
7962026-11-05T17:04:11.450Zuploading: /tmp/debug/dtrace.txt (562778 bytes)
7972026-11-05T17:04:12.514Zuploaded: /tmp/debug/dtrace.txt
7982026-11-05T17:04:12.515Zuploading: /tmp/debug/iostat.txt (153264 bytes)
7992026-11-05T17:04:12.540Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 153264 -> 156575
8002026-11-05T17:04:13.645Zuploaded: /tmp/debug/iostat.txt
8012026-11-05T17:04:13.645Zuploading: /tmp/debug/mpstat.txt (371621 bytes)
8022026-11-05T17:04:13.703Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 371621 -> 380381
8032026-11-05T17:04:14.795Zuploaded: /tmp/debug/mpstat.txt
8042026-11-05T17:04:14.795Zuploading: /tmp/debug/paging.txt (67102 bytes)
8052026-11-05T17:04:14.803Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 67102 -> 68926
8062026-11-05T17:04:15.811Zuploaded: /tmp/debug/paging.txt
8072026-11-05T17:04:15.811Zuploading: /tmp/debug/prstat.txt (700261 bytes)
8082026-11-05T17:04:15.826Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 700261 -> 718355
8092026-11-05T17:04:16.843Zuploaded: /tmp/debug/prstat.txt
8102026-11-05T17:04:16.843Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
8112026-11-05T17:04:17.860Zuploaded: /tmp/debug/psrinfo.txt
8122026-11-05T17:04:17.860Zuploading: /tmp/debug/upstairs-info.txt (101269 bytes)
8132026-11-05T17:04:18.878Zuploaded: /tmp/debug/upstairs-info.txt