01JBM7WNYG1CDJD5QVC4WV0KMD: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JBM7YMD6NPQDX58AMEXBS8KX

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-11-01T16:26:07.409Zjob dependencies complete; ready to run (waiting for 14 m 16 s)
22024-11-01T16:50:27.348Zjob assigned to worker 01JBMA3CV2YQA5FKFR4NA8M6PR (queued for 24 m 19 s)
32024-11-01T16:50:35.831Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-11-01T16:50:37.007Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-11-01T16:50:37.007Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-11-01T16:50:38.086Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-11-01T16:50:38.086Zdownloading input: /input/build/work/bins/crutest.gz
82024-11-01T16:50:39.190Zdownloaded input: /input/build/work/bins/crutest.gz
92024-11-01T16:50:39.192Zdownloading input: /input/build/work/bins/dsc.gz
102024-11-01T16:50:39.858Zdownloaded input: /input/build/work/bins/dsc.gz
112024-11-01T16:50:39.861Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-11-01T16:50:39.880Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-11-01T16:50:39.880Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-11-01T16:50:39.899Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-11-01T16:50:39.899Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-11-01T16:50:39.917Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-11-01T16:50:39.917Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-11-01T16:50:39.936Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-11-01T16:50:39.936Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-11-01T16:50:39.966Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-11-01T16:50:39.966Zdownloading input: /input/build/work/scripts/test_up.sh
222024-11-01T16:50:40.024Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-11-01T16:50:40.025Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-11-01T16:50:40.046Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-11-01T16:50:40.049Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-11-01T16:50:41.101Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-11-01T16:50:41.101Zstarting task 0: "setup"
282024-11-01T16:50:41.112Z++ uname -s
292024-11-01T16:50:41.124Z+ kern=SunOS
302024-11-01T16:50:41.124Z+ case "$kern" in
312024-11-01T16:50:41.125Z+ [[ 0 == 12345 ]]
322024-11-01T16:50:41.125Z+ groupadd -g 12345 build
332024-11-01T16:50:41.125Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
342024-11-01T16:50:43.124Z+ zfs create -o mountpoint=/work rpool/work
352024-11-01T16:50:43.596Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
362024-11-01T16:50:43.598Z+ home_fs=zfs
372024-11-01T16:50:43.598Z+ [[ zfs == autofs ]]
382024-11-01T16:50:43.598Z+ mkdir -p /home/build
392024-11-01T16:50:43.601Z+ chown build:build /home/build /work
402024-11-01T16:50:45.603Z+ chmod 0700 /home/build /work
412024-11-01T16:50:45.606Zprocess exited: duration 4506 ms, exit code 0
 
422024-11-01T16:50:45.613Zstarting task 1: "authentication"
432024-11-01T16:50:45.627Zprocess exited: duration 16 ms, exit code 0
 
442024-11-01T16:50:45.632Zstarting task 2: "build"
452024-11-01T16:50:45.638Z+ banner cores
462024-11-01T16:50:45.661Z
472024-11-01T16:50:45.661Z #### #### ##### ###### ####
482024-11-01T16:50:45.661Z # # # # # # # #
492024-11-01T16:50:45.661Z # # # # # ##### ####
502024-11-01T16:50:45.661Z # # # ##### # #
512024-11-01T16:50:45.661Z # # # # # # # # #
522024-11-01T16:50:45.661Z #### #### # # ###### ####
532024-11-01T16:50:45.661Z
542024-11-01T16:50:45.664Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
552024-11-01T16:50:45.672Z+ echo 'input bins dir contains:'
562024-11-01T16:50:45.672Zinput bins dir contains:
572024-11-01T16:50:45.672Z+ ls -ltr /input/build/work/bins
582024-11-01T16:50:45.674Ztotal 760037
592024-11-01T16:50:45.674Z-rw-r--r-- 1 root root 108929011 Nov 1 16:50 crucible-downstairs.gz
602024-11-01T16:50:45.674Z-rw-r--r-- 1 root root 104573149 Nov 1 16:50 crucible-hammer.gz
612024-11-01T16:50:45.674Z-rw-r--r-- 1 root root 109343961 Nov 1 16:50 crutest.gz
622024-11-01T16:50:45.674Z-rw-r--r-- 1 root root 66055718 Nov 1 16:50 dsc.gz
632024-11-01T16:50:45.674Z+ banner unpack
642024-11-01T16:50:45.677Z
652024-11-01T16:50:45.677Z # # # # ##### ## #### # #
662024-11-01T16:50:45.677Z # # ## # # # # # # # # #
672024-11-01T16:50:45.677Z # # # # # # # # # # ####
682024-11-01T16:50:45.677Z # # # # # ##### ###### # # #
692024-11-01T16:50:45.677Z # # # ## # # # # # # #
702024-11-01T16:50:45.677Z #### # # # # # #### # #
712024-11-01T16:50:45.677Z
722024-11-01T16:50:45.677Z+ mkdir -p /var/tmp/bins
732024-11-01T16:50:45.680Z+ for t in "$input/bins/"*.gz
742024-11-01T16:50:45.680Z++ basename /input/build/work/bins/crucible-downstairs.gz
752024-11-01T16:50:45.682Z+ b=crucible-downstairs.gz
762024-11-01T16:50:45.682Z+ b=crucible-downstairs
772024-11-01T16:50:45.682Z+ gunzip
782024-11-01T16:50:47.919Z+ chmod +x /var/tmp/bins/crucible-downstairs
792024-11-01T16:50:47.922Z+ for t in "$input/bins/"*.gz
802024-11-01T16:50:47.924Z++ basename /input/build/work/bins/crucible-hammer.gz
812024-11-01T16:50:47.924Z+ b=crucible-hammer.gz
822024-11-01T16:50:47.924Z+ b=crucible-hammer
832024-11-01T16:50:47.924Z+ gunzip
842024-11-01T16:50:50.093Z+ chmod +x /var/tmp/bins/crucible-hammer
852024-11-01T16:50:50.095Z+ for t in "$input/bins/"*.gz
862024-11-01T16:50:50.099Z++ basename /input/build/work/bins/crutest.gz
872024-11-01T16:50:50.099Z+ b=crutest.gz
882024-11-01T16:50:50.099Z+ b=crutest
892024-11-01T16:50:50.099Z+ gunzip
902024-11-01T16:50:52.389Z+ chmod +x /var/tmp/bins/crutest
912024-11-01T16:50:52.393Z+ for t in "$input/bins/"*.gz
922024-11-01T16:50:52.393Z++ basename /input/build/work/bins/dsc.gz
932024-11-01T16:50:52.396Z+ b=dsc.gz
942024-11-01T16:50:52.396Z+ b=dsc
952024-11-01T16:50:52.397Z+ gunzip
962024-11-01T16:50:53.787Z+ chmod +x /var/tmp/bins/dsc
972024-11-01T16:50:53.792Z+ export BINDIR=/var/tmp/bins
982024-11-01T16:50:53.792Z+ BINDIR=/var/tmp/bins
992024-11-01T16:50:53.792Z+ jobpid=1133
1002024-11-01T16:50:53.792Z+ echo 'Setup debug logging'
1012024-11-01T16:50:53.792Z+ mkdir /tmp/debug
1022024-11-01T16:50:53.792ZSetup debug logging
1032024-11-01T16:50:53.792Z+ sleep 7200
1042024-11-01T16:50:53.796Z+ psrinfo -v
1052024-11-01T16:50:53.800Z+ df -h
1062024-11-01T16:50:53.821Z+ prstat -d d -mLc 1
1072024-11-01T16:50:53.823Z+ iostat -T d -xn 1
1082024-11-01T16:50:53.823Z+ mpstat -T d 1
1092024-11-01T16:50:53.823Z+ vmstat -T d -p 1
1102024-11-01T16:50:53.827Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1112024-11-01T16:50:53.827Z+ banner test_up_unencrypted
1122024-11-01T16:50:53.827Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1132024-11-01T16:50:53.827Z
1142024-11-01T16:50:53.827Z ##### ###### #### ##### # # ##### # # # #
1152024-11-01T16:50:53.827Z # # # # # # # # # # ## #
1162024-11-01T16:50:53.827Z # ##### #### # # # # # # # # # #
1172024-11-01T16:50:53.827Z # # # # # # ##### # # # # #
1182024-11-01T16:50:53.827Z # # # # # # # # # # # ##
1192024-11-01T16:50:53.827Z # ###### #### # ####### #### # ####### #### # #
1202024-11-01T16:50:53.827Z
1212024-11-01T16:50:53.827Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1222024-11-01T16:50:53.838Z/input/build/work
1232024-11-01T16:50:53.868ZTurn off color for downstairs dump
1242024-11-01T16:50:53.871Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1252024-11-01T16:50:53.871ZCreating 3 downstairs regions
1262024-11-01T16:50:54.231ZStarting 3 downstairs
1272024-11-01T16:50:54.234Zdsc started at PID: 1178
1282024-11-01T16:50:59.405ZDisable automatic restart on all downstairs
1292024-11-01T16:50:59.458Z
1302024-11-01T16:50:59.458ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1312024-11-01T16:50:59.462ZRunning test: span
1322024-11-01T16:51:03.640ZCompleted test: span
1332024-11-01T16:51:08.648ZRunning test: big
1342024-11-01T16:51:12.882ZCompleted test: big
1352024-11-01T16:51:17.890ZRunning test: dep
1362024-11-01T16:51:30.841ZCompleted test: dep
1372024-11-01T16:51:35.849ZRunning test: balloon
1382024-11-01T16:51:49.269ZCompleted test: balloon
1392024-11-01T16:51:54.276ZRunning test: deactivate
1402024-11-01T16:52:48.569ZCompleted test: deactivate
1412024-11-01T16:52:53.577ZRunning hammer
1422024-11-01T16:52:57.165ZRun repair tests
1432024-11-01T16:52:57.169Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1442024-11-01T16:52:57.228Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.499625998Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"task":"crutest"}
1452024-11-01T16:52:57.230Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.500225047Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"task":"crutest"}
1462024-11-01T16:52:57.230Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.501946894Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"task":"crutest"}
1472024-11-01T16:52:57.232Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.502131412Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215}
1482024-11-01T16:52:57.232Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.502176287Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215}
1492024-11-01T16:52:57.233Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.502191775Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215}
1502024-11-01T16:52:57.233Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.502206334Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215}
1512024-11-01T16:52:57.233Z{"msg":"Crucible c493296b-2009-41d4-b218-a33165daa232 has session id: 356afd17-04ea-40f7-9a15-ecf2921d4d48","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.502645982Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1522024-11-01T16:52:57.233Z{"msg":"Upstairs opts: Upstairs UUID: c493296b-2009-41d4-b218-a33165daa232, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.502690536Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1532024-11-01T16:52:57.233Z{"msg":"Crucible stats registered with UUID: c493296b-2009-41d4-b218-a33165daa232","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.502710624Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1542024-11-01T16:52:57.265Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.536661868Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"task":"crutest"}
1552024-11-01T16:52:57.265Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.536721021Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215}
1562024-11-01T16:52:57.268Z{"msg":"c493296b-2009-41d4-b218-a33165daa232 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.536770455Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1572024-11-01T16:52:57.268Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.536814619Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"0","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1582024-11-01T16:52:57.268Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.536868303Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"0","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1592024-11-01T16:52:57.268Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.536915407Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"1","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1602024-11-01T16:52:57.268Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.536971491Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"1","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1612024-11-01T16:52:57.268Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.537037653Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"2","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1622024-11-01T16:52:57.268Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.537071499Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"2","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1632024-11-01T16:52:57.268Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.537126023Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1642024-11-01T16:52:57.268Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.537172577Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1652024-11-01T16:52:57.268Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.537206653Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1662024-11-01T16:52:57.268Z{"msg":"ds_connection connected from Ok(127.0.0.1:52703)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.537240949Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1672024-11-01T16:52:57.268Z{"msg":"ds_connection connected from Ok(127.0.0.1:50769)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.537292213Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1682024-11-01T16:52:57.268Z{"msg":"ds_connection connected from Ok(127.0.0.1:52045)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.537343267Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1692024-11-01T16:52:57.268Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.538046464Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"0","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1702024-11-01T16:52:57.268Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.538076661Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"0","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1712024-11-01T16:52:57.268Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.538095178Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"1","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1722024-11-01T16:52:57.268Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.538136194Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"1","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1732024-11-01T16:52:57.268Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.5381634Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"2","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1742024-11-01T16:52:57.268Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.538185968Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"2","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1752024-11-01T16:52:57.268Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.5387632Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"2","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1762024-11-01T16:52:57.268Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.538941888Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"1","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1772024-11-01T16:52:57.268Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.538971915Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"0","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1782024-11-01T16:52:57.268Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539132416Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"2","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1792024-11-01T16:52:57.268Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539170521Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"2","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1802024-11-01T16:52:57.268Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539186579Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1812024-11-01T16:52:57.271Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539330962Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"1","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1822024-11-01T16:52:57.271Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539370858Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"1","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1832024-11-01T16:52:57.271Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539386626Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1842024-11-01T16:52:57.271Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539457678Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"0","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1852024-11-01T16:52:57.272Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539485134Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"0","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1862024-11-01T16:52:57.272Z{"msg":"[0]R flush_numbers: [1537, 1537, 1537, 1537, 1537]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539499733Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1872024-11-01T16:52:57.272Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.53952024Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1882024-11-01T16:52:57.272Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539534218Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1892024-11-01T16:52:57.272Z{"msg":"[1]R flush_numbers: [1537, 1537, 1537, 1537, 1537]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539554496Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1902024-11-01T16:52:57.272Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539574694Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1912024-11-01T16:52:57.272Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539588392Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1922024-11-01T16:52:57.272Z{"msg":"[2]R flush_numbers: [1537, 1537, 1537, 1537, 1537]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.53960227Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1932024-11-01T16:52:57.272Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539615539Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1942024-11-01T16:52:57.272Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539635917Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1952024-11-01T16:52:57.272Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539649205Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1962024-11-01T16:52:57.272Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539662393Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1972024-11-01T16:52:57.272Z{"msg":"Next flush: 1538","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539682321Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
1982024-11-01T16:52:57.273Z{"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1992024-11-01T16:52:57.273Z,"time":"2024-11-01T16:52:56.539695499Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"":"downstairs","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
2002024-11-01T16:52:57.273Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539717417Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
2012024-11-01T16:52:57.273Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539729775Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"0","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
2022024-11-01T16:52:57.273Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539749723Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"1","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
2032024-11-01T16:52:57.273Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539763042Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"client":"2","":"downstairs","session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
2042024-11-01T16:52:57.273Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.53977626Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
2052024-11-01T16:52:57.273Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539796778Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
2062024-11-01T16:52:57.273Z{"msg":"c493296b-2009-41d4-b218-a33165daa232 is now active with session: 356afd17-04ea-40f7-9a15-ecf2921d4d48","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539809836Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
2072024-11-01T16:52:57.273Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539821824Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215,"session_id":"356afd17-04ea-40f7-9a15-ecf2921d4d48"}
2082024-11-01T16:52:57.273Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:52:56.539841222Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215}
2092024-11-01T16:52:57.273ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2102024-11-01T16:52:57.273ZFill test
2112024-11-01T16:52:57.279ZRead and Verify all blocks (0..50 range:false)
2122024-11-01T16:52:57.286ZWrote out file "/var/tmp/test_up-build/verify_file"
2132024-11-01T16:52:57.286ZCLIENT: Tests done. All submitted work has been ACK'd
2142024-11-01T16:52:57.288Z----------------------------------------------------------------
2152024-11-01T16:52:57.289Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2162024-11-01T16:52:57.289ZStates: Active Active Active
2172024-11-01T16:52:57.289ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2182024-11-01T16:52:57.289Z 1002 Acked Read 50 Done Done Done false
2192024-11-01T16:52:57.289Z STATES DS:0 DS:1 DS:2 TOTAL
2202024-11-01T16:52:57.289Z Sent 0 0 0 0
2212024-11-01T16:52:57.290Z Done 1 1 1 3
2222024-11-01T16:52:57.290Z Skipped 0 0 0 0
2232024-11-01T16:52:57.290Z Error 0 0 0 0
2242024-11-01T16:52:57.290ZLast Flush: 1001 1001 1001
2252024-11-01T16:52:57.290ZDownstairs last five completed: 1001 1000
2262024-11-01T16:52:57.290ZUpstairs last five completed: 1002 1001 1000
2272024-11-01T16:52:57.290ZCLIENT: Up:0 ds:1 act:3
2282024-11-01T16:53:01.288Z----------------------------------------------------------------
2292024-11-01T16:53:01.289Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2302024-11-01T16:53:01.289ZDownstairs last five completed: 1003 1002 1001 1000
2312024-11-01T16:53:01.289ZUpstairs last five completed: 1003 1002 1001 1000
2322024-11-01T16:53:01.289ZCLIENT: Up:0 ds:0 act:3
2332024-11-01T16:53:01.289ZCLIENT: All crucible jobs finished, exiting program
2342024-11-01T16:53:01.291Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:00.559436271Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1215}
2352024-11-01T16:53:01.299ZRepair setup passed
2362024-11-01T16:53:01.303ZCopy the region for /var/tmp/test_up-build/8830
2372024-11-01T16:53:01.305Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2382024-11-01T16:53:01.308Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2392024-11-01T16:53:01.366Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.636851805Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"task":"crutest"}
2402024-11-01T16:53:01.368Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.63740153Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"task":"crutest"}
2412024-11-01T16:53:01.368Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.639189849Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"task":"crutest"}
2422024-11-01T16:53:01.368Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.639380877Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221}
2432024-11-01T16:53:01.368Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.63943664Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221}
2442024-11-01T16:53:01.368Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.639464847Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221}
2452024-11-01T16:53:01.369Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.639484495Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221}
2462024-11-01T16:53:01.371Z{"msg":"Crucible 2d01cf4b-7026-47e4-bf29-b95b14c912b9 has session id: 3cec34ee-a943-450e-8c47-5402f6fa4962","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.63994772Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2472024-11-01T16:53:01.372Z{"msg":"Upstairs opts: Upstairs UUID: 2d01cf4b-7026-47e4-bf29-b95b14c912b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.640024611Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2482024-11-01T16:53:01.372Z{"msg":"Crucible stats registered with UUID: 2d01cf4b-7026-47e4-bf29-b95b14c912b9","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.640063316Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2492024-11-01T16:53:01.406Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.67409919Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"task":"crutest"}
2502024-11-01T16:53:01.406Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674164032Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221}
2512024-11-01T16:53:01.406Z{"msg":"2d01cf4b-7026-47e4-bf29-b95b14c912b9 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674206277Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2522024-11-01T16:53:01.410Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674249832Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"0","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2532024-11-01T16:53:01.410Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674308825Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"0","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2542024-11-01T16:53:01.410Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674364428Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"1","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2552024-11-01T16:53:01.410Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674416672Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"1","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2562024-11-01T16:53:01.410Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674450778Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"2","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2572024-11-01T16:53:01.410Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674491013Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"2","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2582024-11-01T16:53:01.410Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674531029Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2592024-11-01T16:53:01.410Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674564365Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2602024-11-01T16:53:01.410Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674592932Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2612024-11-01T16:53:01.410Z{"msg":"ds_connection connected from Ok(127.0.0.1:49976)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674753422Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2622024-11-01T16:53:01.410Z{"msg":"ds_connection connected from Ok(127.0.0.1:59950)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674849351Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2632024-11-01T16:53:01.410Z{"msg":"ds_connection connected from Ok(127.0.0.1:38363)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.674901065Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2642024-11-01T16:53:01.410Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.67545327Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"0","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2652024-11-01T16:53:01.410Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.675494455Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"0","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2662024-11-01T16:53:01.410Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.675531181Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"2","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2672024-11-01T16:53:01.410Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.675590644Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"2","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2682024-11-01T16:53:01.410Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.675646237Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"1","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2692024-11-01T16:53:01.410Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.675736766Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"1","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2702024-11-01T16:53:01.410Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676222149Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"1","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2712024-11-01T16:53:01.410Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676263904Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"2","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2722024-11-01T16:53:01.410Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676282042Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"0","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2732024-11-01T16:53:01.410Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676658378Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"1","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2742024-11-01T16:53:01.410Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676709462Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"1","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2752024-11-01T16:53:01.410Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676751197Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2762024-11-01T16:53:01.410Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676787093Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"2","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2772024-11-01T16:53:01.410Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676821929Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"2","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2782024-11-01T16:53:01.411Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676844926Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2792024-11-01T16:53:01.411Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676877652Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"0","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2802024-11-01T16:53:01.411Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676922347Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"0","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2812024-11-01T16:53:01.411Z{"msg":"[0]R flush_numbers: [1538, 1538, 1538, 1538, 1538]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676950263Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2822024-11-01T16:53:01.411Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.676987579Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2832024-11-01T16:53:01.411Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677026344Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2842024-11-01T16:53:01.411ZWait for a query_work_queue command to finish before sending IO
2852024-11-01T16:53:01.411Z{"msg":"[1]R flush_numbers: [1538, 1538, 1538, 1538, 1538]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677073999Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2862024-11-01T16:53:01.411Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677112364Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2872024-11-01T16:53:01.411Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.67715114Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2882024-11-01T16:53:01.411Z{"msg":"[2]R flush_numbers: [1538, 1538, 1538, 1538, 1538]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677191415Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2892024-11-01T16:53:01.411Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677238619Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2902024-11-01T16:53:01.411Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677279864Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2912024-11-01T16:53:01.411Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2922024-11-01T16:53:01.411Z,"time":"2024-11-01T16:53:00.67731797Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2932024-11-01T16:53:01.411Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677347247Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2942024-11-01T16:53:01.411Z{"msg":"Next flush: 1539","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677372344Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2952024-11-01T16:53:01.411Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677386232Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"":"downstairs","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2962024-11-01T16:53:01.411Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677406709Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2972024-11-01T16:53:01.411Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677419028Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"0","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2982024-11-01T16:53:01.411Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677438986Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"1","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
2992024-11-01T16:53:01.411Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677452874Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"client":"2","":"downstairs","session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
3002024-11-01T16:53:01.412Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677472842Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
3012024-11-01T16:53:01.412Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":Loading write count information from file "/var/tmp/test_up-build/verify_file"
3022024-11-01T16:53:01.412Z30,"time":"2024-11-01T16:53:00.677493499Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
3032024-11-01T16:53:01.412Z{"msg":"2d01cf4b-7026-47e4-bf29-b95b14c912b9 is now active with session: 3cec34ee-a943-450e-8c47-5402f6fa4962","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677513217Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
3042024-11-01T16:53:01.412Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677525706Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221,"session_id":"3cec34ee-a943-450e-8c47-5402f6fa4962"}
3052024-11-01T16:53:01.412ZFill test
3062024-11-01T16:53:01.412Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:00.677545943Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221}
3072024-11-01T16:53:01.421ZRead and Verify all blocks (0..50 range:false)
3082024-11-01T16:53:01.426ZWrote out file "/var/tmp/test_up-build/verify_file"
3092024-11-01T16:53:01.426ZCLIENT: Tests done. All submitted work has been ACK'd
3102024-11-01T16:53:01.428Z----------------------------------------------------------------
3112024-11-01T16:53:01.428Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3122024-11-01T16:53:01.428ZStates: Active Active Active
3132024-11-01T16:53:01.428ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3142024-11-01T16:53:01.428Z 1002 Acked Read 50 Done Done Done false
3152024-11-01T16:53:01.428Z STATES DS:0 DS:1 DS:2 TOTAL
3162024-11-01T16:53:01.428Z Sent 0 0 0 0
3172024-11-01T16:53:01.428Z Done 1 1 1 3
3182024-11-01T16:53:01.429Z Skipped 0 0 0 0
3192024-11-01T16:53:01.429Z Error 0 0 0 0
3202024-11-01T16:53:01.429ZLast Flush: 1001 1001 1001
3212024-11-01T16:53:01.429ZDownstairs last five completed: 1001 1000
3222024-11-01T16:53:01.429ZUpstairs last five completed: 1002 1001 1000
3232024-11-01T16:53:01.429ZCLIENT: Up:0 ds:1 act:3
3242024-11-01T16:53:05.430Z----------------------------------------------------------------
3252024-11-01T16:53:05.430Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3262024-11-01T16:53:05.430ZDownstairs last five completed: 1003 1002 1001 1000
3272024-11-01T16:53:05.430ZUpstairs last five completed: 1003 1002 1001 1000
3282024-11-01T16:53:05.430ZCLIENT: Up:0 ds:0 act:3
3292024-11-01T16:53:05.430ZCLIENT: All crucible jobs finished, exiting program
3302024-11-01T16:53:05.433Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:04.696651999Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1221}
3312024-11-01T16:53:05.435ZRepair part 1 passed
3322024-11-01T16:53:05.435Z
3332024-11-01T16:53:05.435ZKill the current downstairs
3342024-11-01T16:53:05.597ZDownstairs 2 stopped
3352024-11-01T16:53:05.597Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3362024-11-01T16:53:05.600ZNow put back the original so we have a mismatch
3372024-11-01T16:53:05.600Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3382024-11-01T16:53:05.602ZRestart downstairs with old directory
3392024-11-01T16:53:05.655Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3402024-11-01T16:53:05.672ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3412024-11-01T16:53:05.674Z 0 000-009 32 32 31 1539 1539 1538 F F F <---
3422024-11-01T16:53:05.674Z 1 010-019 32 32 31 1539 1539 1538 F F F <---
3432024-11-01T16:53:05.674Z 2 020-029 32 32 31 1539 1539 1538 F F F <---
3442024-11-01T16:53:05.674Z 3 030-039 32 32 31 1539 1539 1538 F F F <---
3452024-11-01T16:53:05.674Z 4 040-049 32 32 31 1539 1539 1538 F F F <---
3462024-11-01T16:53:05.674ZMax gen: 32, Max flush: 1539
3472024-11-01T16:53:05.674ZError: Difference in extent metadata found!
3482024-11-01T16:53:05.677Zdump test found error as expected
3492024-11-01T16:53:05.677Z
3502024-11-01T16:53:05.677Z
3512024-11-01T16:53:05.677Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3522024-11-01T16:53:05.735Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.005479925Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"task":"crutest"}
3532024-11-01T16:53:05.738Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.006055037Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"task":"crutest"}
3542024-11-01T16:53:05.738Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.007796682Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"task":"crutest"}
3552024-11-01T16:53:05.738Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.007956683Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229}
3562024-11-01T16:53:05.738Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"3818f0fc76fe787c6cc412af30fd59b930203885\",\n git_commit_timestamp: \"2024-11-01T16:08:30.000000000Z\",\n git_branch: \"renovate/hyper-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.008001048Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229}
3572024-11-01T16:53:05.739Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.008024235Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229}
3582024-11-01T16:53:05.739Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.008039843Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229}
3592024-11-01T16:53:05.742Z{"msg":"Crucible 3c947f83-f2fc-4a8e-bd5d-b5ff281c64bd has session id: f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.008528525Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3602024-11-01T16:53:05.742Z{"msg":"Upstairs opts: Upstairs UUID: 3c947f83-f2fc-4a8e-bd5d-b5ff281c64bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.008593418Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3612024-11-01T16:53:05.742Z{"msg":"Crucible stats registered with UUID: 3c947f83-f2fc-4a8e-bd5d-b5ff281c64bd","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.008651681Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3622024-11-01T16:53:05.773Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.0434856Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"task":"crutest"}
3632024-11-01T16:53:05.773Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043537294Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229}
3642024-11-01T16:53:05.776Z{"msg":"3c947f83-f2fc-4a8e-bd5d-b5ff281c64bd active request set","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043584688Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3652024-11-01T16:53:05.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043633912Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3662024-11-01T16:53:05.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043696265Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3672024-11-01T16:53:05.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043732011Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3682024-11-01T16:53:05.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043771636Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3692024-11-01T16:53:05.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043803903Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3702024-11-01T16:53:05.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043846638Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3712024-11-01T16:53:05.776Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043889173Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3722024-11-01T16:53:05.776Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043916719Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3732024-11-01T16:53:05.776Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.043954315Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3742024-11-01T16:53:05.776Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:05.043988931Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3752024-11-01T16:53:05.776Z{"msg":"ds_connection connected from Ok(127.0.0.1:61124)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.044020167Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3762024-11-01T16:53:05.777Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:05.044055213Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3772024-11-01T16:53:05.777Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:05.044089959Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3782024-11-01T16:53:05.777Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.044232922Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3792024-11-01T16:53:05.777Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.044260169Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3802024-11-01T16:53:05.777Z{"msg":"ds_connection connected from Ok(127.0.0.1:44973)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.044295194Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3812024-11-01T16:53:05.777Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.044312292Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3822024-11-01T16:53:05.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.044609428Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3832024-11-01T16:53:05.777Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.044658582Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3842024-11-01T16:53:05.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.044699657Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3852024-11-01T16:53:05.777Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.044730483Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3862024-11-01T16:53:05.777Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.04526284Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3872024-11-01T16:53:05.777Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.045301206Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3882024-11-01T16:53:05.778Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.045681471Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3892024-11-01T16:53:05.778Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.045732215Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3902024-11-01T16:53:05.778Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.045751982Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3912024-11-01T16:53:05.778Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.04577492Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3922024-11-01T16:53:05.778Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.045789698Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3932024-11-01T16:53:05.778Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:05.045810356Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3942024-11-01T16:53:15.776Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.044535188Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3952024-11-01T16:53:15.779Z{"msg":"ds_connection connected from Ok(127.0.0.1:52242)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.044678141Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3962024-11-01T16:53:15.779Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.045684063Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3972024-11-01T16:53:15.779Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.045735497Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3982024-11-01T16:53:15.779Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046269234Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
3992024-11-01T16:53:15.779Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.04663589Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4002024-11-01T16:53:15.779Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046666816Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4012024-11-01T16:53:15.779Z{"msg":"[0]R flush_numbers: [1539, 1539, 1539, 1539, 1539]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046682445Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4022024-11-01T16:53:15.779Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046704742Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4032024-11-01T16:53:15.779Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046735888Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4042024-11-01T16:53:15.779Z{"msg":"[1]R flush_numbers: [1539, 1539, 1539, 1539, 1539]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046751526Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4052024-11-01T16:53:15.779Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046784473Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4062024-11-01T16:53:15.779Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.04680454Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4072024-11-01T16:53:15.779Z{"msg":"[2]R flush_numbers: [1538, 1538, 1538, 1538, 1538]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046836336Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4082024-11-01T16:53:15.779Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046853465Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4092024-11-01T16:53:15.780Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046885081Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4102024-11-01T16:53:15.780Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046902689Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4112024-11-01T16:53:15.780Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046933755Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4122024-11-01T16:53:15.780Z{"msg":"Next flush: 1540","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046949193Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4132024-11-01T16:53:15.780Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046965781Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4142024-11-01T16:53:15.780Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.046997357Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4152024-11-01T16:53:15.780Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047015775Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4162024-11-01T16:53:15.780Z{"msg":"extent:0 flush: 1539 1539 1538 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047048622Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4172024-11-01T16:53:15.780Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047067419Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4182024-11-01T16:53:15.780Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047099665Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4192024-11-01T16:53:15.780Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047117543Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4202024-11-01T16:53:15.780Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047151219Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4212024-11-01T16:53:15.783Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047170527Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4222024-11-01T16:53:15.783Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047220221Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4232024-11-01T16:53:15.783Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047254917Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4242024-11-01T16:53:15.784Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047273955Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4252024-11-01T16:53:15.784Z{"msg":"extent:1 flush: 1539 1539 1538 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.04731481Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4262024-11-01T16:53:15.784Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047356155Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4272024-11-01T16:53:15.784Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.0473989Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4282024-11-01T16:53:15.784Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047429397Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4292024-11-01T16:53:15.784Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047463013Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4302024-11-01T16:53:15.784Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047479381Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4312024-11-01T16:53:15.784Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047501508Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4322024-11-01T16:53:15.784Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047515667Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4332024-11-01T16:53:15.785Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047536444Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4342024-11-01T16:53:15.785Z{"msg":"extent:2 flush: 1539 1539 1538 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047551452Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4352024-11-01T16:53:15.785Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.04757268Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4362024-11-01T16:53:15.785Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047587508Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4372024-11-01T16:53:15.785Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047610205Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4382024-11-01T16:53:15.785Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047628633Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4392024-11-01T16:53:15.786Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.04765673Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4402024-11-01T16:53:15.786Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047680067Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4412024-11-01T16:53:15.786Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047709424Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4422024-11-01T16:53:15.786Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047725312Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4432024-11-01T16:53:15.786Z{"msg":"extent:3 flush: 1539 1539 1538 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.04774084Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4442024-11-01T16:53:15.786Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047755918Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4452024-11-01T16:53:15.786Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047778376Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4462024-11-01T16:53:15.786Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047793334Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4472024-11-01T16:53:15.786Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047814801Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4482024-11-01T16:53:15.786Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047829249Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4492024-11-01T16:53:15.787Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047844628Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4502024-11-01T16:53:15.787Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047858396Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4512024-11-01T16:53:15.787Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047889342Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4522024-11-01T16:53:15.787Z{"msg":"extent:4 flush: 1539 1539 1538 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047905201Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4532024-11-01T16:53:15.787Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047927758Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4542024-11-01T16:53:15.787Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047942436Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4552024-11-01T16:53:15.787Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047957024Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4562024-11-01T16:53:15.787Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.047978412Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4572024-11-01T16:53:15.787Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.04799281Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4582024-11-01T16:53:15.787Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.048018057Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4592024-11-01T16:53:15.787Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.048032186Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4602024-11-01T16:53:15.787Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.048053633Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4612024-11-01T16:53:15.787Z{"msg":"starting reconciliation 01631b50-4465-4e0f-a162-2785c0b57f3a: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.048067851Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4622024-11-01T16:53:15.787Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.048091928Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4632024-11-01T16:53:15.787Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1540, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1540, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1540, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1540, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1540, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.048169979Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4642024-11-01T16:53:15.788Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.048185187Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4652024-11-01T16:53:15.788Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.048200456Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4662024-11-01T16:53:15.788Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.048214934Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4672024-11-01T16:53:15.788Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.048236701Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4682024-11-01T16:53:15.788Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.04825065Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4692024-11-01T16:53:15.791Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.060389447Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4702024-11-01T16:53:15.793Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.060854562Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4712024-11-01T16:53:15.794Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.061167945Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4722024-11-01T16:53:15.794Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.061509785Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4732024-11-01T16:53:15.794Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.061565488Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4742024-11-01T16:53:15.800Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.069466546Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4752024-11-01T16:53:15.803Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.069915253Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4762024-11-01T16:53:15.803Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.070229976Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4772024-11-01T16:53:15.803Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.070541459Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4782024-11-01T16:53:15.803Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.070586684Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4792024-11-01T16:53:15.809Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.078399392Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4802024-11-01T16:53:15.812Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.078792985Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4812024-11-01T16:53:15.812Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.079059324Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4822024-11-01T16:53:15.812Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.079375347Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4832024-11-01T16:53:15.812Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.079417692Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4842024-11-01T16:53:15.817Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.086266254Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4852024-11-01T16:53:15.819Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.08663401Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4862024-11-01T16:53:15.820Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.086903768Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4872024-11-01T16:53:15.820Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.087148219Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4882024-11-01T16:53:15.820Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.087194244Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4892024-11-01T16:53:15.826Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.094801166Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4902024-11-01T16:53:15.828Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.095177222Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4912024-11-01T16:53:15.828Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.095222466Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"0","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4922024-11-01T16:53:15.828Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.095243694Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"1","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4932024-11-01T16:53:15.829Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.095285939Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"client":"2","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4942024-11-01T16:53:15.829ZWait for a query_work_queue command to finish before sending IO
4952024-11-01T16:53:15.829Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.095331354Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4962024-11-01T16:53:15.829Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.095380138Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4972024-11-01T16:53:15.829Z{"msg":"3c947f83-f2fc-4a8e-bd5d-b5ff281c64bd is now active with session: f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.095428042Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
4982024-11-01T16:53:15.829Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-11-01T16:53:15.095474207Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229}
4992024-11-01T16:53:15.829Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-11-01T16:53:15.095506543Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229,"reconcile":"01631b50-4465-4e0f-a162-2785c0b57f3a","":"downstairs","":"downstairs","session_id":"f7e30c2a-4f8a-4e6f-a6d8-d0b710e74e28"}
5002024-11-01T16:53:15.829ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
5012024-11-01T16:53:15.829ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
5022024-11-01T16:53:15.829ZRead and Verify all blocks (0..50 range:true)
5032024-11-01T16:53:15.833ZWrote out file "/var/tmp/test_up-build/verify_file"
5042024-11-01T16:53:15.833ZVerify test completed
5052024-11-01T16:53:15.836ZWrote out file "/var/tmp/test_up-build/verify_file"
5062024-11-01T16:53:15.836ZCLIENT: Tests done. All submitted work has been ACK'd
5072024-11-01T16:53:15.836Z----------------------------------------------------------------
5082024-11-01T16:53:15.836Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5092024-11-01T16:53:15.836ZStates: Active Active Active
5102024-11-01T16:53:15.836ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5112024-11-01T16:53:15.836Z 1000 Acked Read 50 Done Done Done false
5122024-11-01T16:53:15.836Z STATES DS:0 DS:1 DS:2 TOTAL
5132024-11-01T16:53:15.836Z Sent 0 0 0 0
5142024-11-01T16:53:15.837Z Done 1 1 1 3
5152024-11-01T16:53:15.837Z Skipped 0 0 0 0
5162024-11-01T16:53:15.837Z Error 0 0 0 0
5172024-11-01T16:53:15.837ZLast Flush: 0 0 0
5182024-11-01T16:53:15.837ZDownstairs last five completed:
5192024-11-01T16:53:15.837ZUpstairs last five completed: 1000
5202024-11-01T16:53:15.837ZCLIENT: Up:0 ds:1 act:3
5212024-11-01T16:53:19.836Z----------------------------------------------------------------
5222024-11-01T16:53:19.836Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5232024-11-01T16:53:19.836ZDownstairs last five completed: 1001 1000
5242024-11-01T16:53:19.836ZUpstairs last five completed: 1001 1000
5252024-11-01T16:53:19.836ZCLIENT: Up:0 ds:0 act:3
5262024-11-01T16:53:19.836ZCLIENT: All crucible jobs finished, exiting program
5272024-11-01T16:53:19.839Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-01T16:53:19.104188939Z","hostname":"w-01JBMA3CV2YQA5FKFR4NA8M6PR","pid":1229}
5282024-11-01T16:53:19.843ZRepair part 2 passed
5292024-11-01T16:53:19.845Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5302024-11-01T16:53:19.860ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5312024-11-01T16:53:19.860Z 0 000-009 32 32 32 1539 1539 1539 F F F
5322024-11-01T16:53:19.860Z 1 010-019 32 32 32 1539 1539 1539 F F F
5332024-11-01T16:53:19.860Z 2 020-029 32 32 32 1539 1539 1539 F F F
5342024-11-01T16:53:19.860Z 3 030-039 32 32 32 1539 1539 1539 F F F
5352024-11-01T16:53:19.862Z 4 040-049 32 32 32 1539 1539 1539 F F F
5362024-11-01T16:53:19.863ZMax gen: 32, Max flush: 1539
5372024-11-01T16:53:19.865Zdump test passed
5382024-11-01T16:53:19.865Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5392024-11-01T16:53:19.880Z Extent 1
5402024-11-01T16:53:19.880ZGEN 32 32 32
5412024-11-01T16:53:19.880ZFLUSH_ID 1539 1539 1539
5422024-11-01T16:53:19.880ZDIRTY
5432024-11-01T16:53:19.880Z
5442024-11-01T16:53:19.880ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5452024-11-01T16:53:19.882Z010 A A A A A A
5462024-11-01T16:53:19.884Z011 A A A A A A
5472024-11-01T16:53:19.884Z012 A A A A A A
5482024-11-01T16:53:19.887Z013 A A A A A A
5492024-11-01T16:53:19.887Z014 A A A A A A
5502024-11-01T16:53:19.889Z015 A A A A A A
5512024-11-01T16:53:19.889Z016 A A A A A A
5522024-11-01T16:53:19.893Z017 A A A A A A
5532024-11-01T16:53:19.893Z018 A A A A A A
5542024-11-01T16:53:19.895Z019 A A A A A A
5552024-11-01T16:53:19.897Zdump extent test passed
5562024-11-01T16:53:19.897Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5572024-11-01T16:53:19.911ZExtent 2 Block in extent 0 Actual block 20
5582024-11-01T16:53:19.911Z
5592024-11-01T16:53:19.913Z DATA SHA256 VER
5602024-11-01T16:53:19.913Z------ ---------------------------------------------------------------- ---
5612024-11-01T16:53:19.913Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5622024-11-01T16:53:19.913Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5632024-11-01T16:53:19.916Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5642024-11-01T16:53:19.916Z
5652024-11-01T16:53:19.916ZNONCES 0 1 2 DIFF
5662024-11-01T16:53:19.916Z------ ------------------------ ------------------------ ------------------------ -----
5672024-11-01T16:53:19.916Z
5682024-11-01T16:53:19.916Z TAGS 0 1 2 DIFF
5692024-11-01T16:53:19.916Z------ -------------------------------- -------------------------------- -------------------------------- -----
5702024-11-01T16:53:19.916Z
5712024-11-01T16:53:19.916ZHASHES 0 1 2 DIFF
5722024-11-01T16:53:19.916Z------ ---------------- ---------------- ---------------- -----
5732024-11-01T16:53:19.916Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5742024-11-01T16:53:19.916Z
5752024-11-01T16:53:19.918Zdump block test passed
5762024-11-01T16:53:19.918ZInitial upstairs tests have completed, stopping all downstairs
5772024-11-01T16:53:24.979ZCreating 4 larger downstairs regions
5782024-11-01T16:53:25.476ZStarting 4 downstairs
5792024-11-01T16:53:30.484Zdsc restarted at PID: 1242
5802024-11-01T16:53:30.712ZNow do the replace-reconcile test
5812024-11-01T16:53:30.770ZUsing 8840 for the replacement port
5822024-11-01T16:55:56.237ZCompleted test: replace-reconcile
5832024-11-01T16:55:56.240ZNow do the replace-before-active test
5842024-11-01T16:58:32.864ZCompleted test: replace-before-active
5852024-11-01T16:58:32.868ZAll tests have completed, stopping all downstairs
5862024-11-01T16:58:32.920Z
5872024-11-01T16:58:32.920ZAll Tests have passed
5882024-11-01T16:58:32.923Z7:39 Test duration
5892024-11-01T16:58:32.923Z
5902024-11-01T16:58:32.923Zreal 7:39.028521554
5912024-11-01T16:58:32.923Zuser 1:03.607718356
5922024-11-01T16:58:32.923Zsys 11.862186342
5932024-11-01T16:58:32.923Ztrap 0.146569363
5942024-11-01T16:58:32.923Ztflt 0.012773302
5952024-11-01T16:58:32.923Zdflt 0.056183182
5962024-11-01T16:58:32.923Zkflt 0.001144925
5972024-11-01T16:58:32.923Zlock 5:46:27.374305990
5982024-11-01T16:58:32.923Zslp 24:36.237537052
5992024-11-01T16:58:32.924Zlat 9.146781767
6002024-11-01T16:58:32.924Zstop 0.144054891
6012024-11-01T16:58:32.924Z+ echo 'test-up-unencrypted ends'
6022024-11-01T16:58:32.924Ztest-up-unencrypted ends
6032024-11-01T16:58:37.927Zprocess exited: duration 467226 ms, exit code 0
6042024-11-01T16:58:37.927Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6052024-11-01T16:59:37.981Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6062024-11-01T16:59:37.981Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6072024-11-01T16:59:37.991Zfound 14 output files
6082024-11-01T16:59:37.991Zuploading: /tmp/test_up-build/dsc-out.txt (37040 bytes)
6092024-11-01T16:59:39.006Zuploaded: /tmp/test_up-build/dsc-out.txt
6102024-11-01T16:59:39.006Zuploading: /tmp/test_up-build/test_up_out.txt (10938952 bytes)
6112024-11-01T16:59:40.191Zuploaded: /tmp/test_up-build/test_up_out.txt
6122024-11-01T16:59:40.193Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (253707 bytes)
6132024-11-01T16:59:41.204Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6142024-11-01T16:59:41.205Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144651 bytes)
6152024-11-01T16:59:42.216Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6162024-11-01T16:59:42.216Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251106 bytes)
6172024-11-01T16:59:43.229Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6182024-11-01T16:59:43.232Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139421 bytes)
6192024-11-01T16:59:44.241Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6202024-11-01T16:59:44.244Zuploading: /tmp/debug/df.txt (1270 bytes)
6212024-11-01T16:59:45.250Zuploaded: /tmp/debug/df.txt
6222024-11-01T16:59:45.252Zuploading: /tmp/debug/dtrace.txt (418682 bytes)
6232024-11-01T16:59:46.266Zuploaded: /tmp/debug/dtrace.txt
6242024-11-01T16:59:46.269Zuploading: /tmp/debug/iostat.txt (158108 bytes)
6252024-11-01T16:59:46.273Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 158108 -> 160516
6262024-11-01T16:59:47.278Zuploaded: /tmp/debug/iostat.txt
6272024-11-01T16:59:47.278Zuploading: /tmp/debug/mpstat.txt (383258 bytes)
6282024-11-01T16:59:47.290Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 383258 -> 389828
6292024-11-01T16:59:48.296Zuploaded: /tmp/debug/mpstat.txt
6302024-11-01T16:59:48.299Zuploading: /tmp/debug/paging.txt (69274 bytes)
6312024-11-01T16:59:48.303Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 69274 -> 70592
6322024-11-01T16:59:49.308Zuploaded: /tmp/debug/paging.txt
6332024-11-01T16:59:49.311Zuploading: /tmp/debug/prstat.txt (722185 bytes)
6342024-11-01T16:59:49.326Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 722185 -> 736416
6352024-11-01T16:59:50.334Zuploaded: /tmp/debug/prstat.txt
6362024-11-01T16:59:50.334Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6372024-11-01T16:59:51.342Zuploaded: /tmp/debug/psrinfo.txt
6382024-11-01T16:59:51.342Zuploading: /tmp/debug/upstairs-info.txt (70468 bytes)
6392024-11-01T16:59:52.352Zuploaded: /tmp/debug/upstairs-info.txt