01JC6P5CB7QRRTS19AK9PFF9TM: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JC6P5X2J3M6SFS94N08A5Q5G

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-11-08T20:21:01.397Zjob dependencies complete; ready to run (waiting for 14 m 12 s)
22024-11-08T20:22:06.787Zjob assigned to worker 01JC6Q02X5E25PDH9N1636SCK7 (queued for 1 m 5 s)
32024-11-08T20:22:15.237Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-11-08T20:22:16.749Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-11-08T20:22:16.749Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-11-08T20:22:17.816Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-11-08T20:22:17.816Zdownloading input: /input/build/work/bins/crutest.gz
82024-11-08T20:22:18.928Zdownloaded input: /input/build/work/bins/crutest.gz
92024-11-08T20:22:18.928Zdownloading input: /input/build/work/bins/dsc.gz
102024-11-08T20:22:19.615Zdownloaded input: /input/build/work/bins/dsc.gz
112024-11-08T20:22:19.617Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-11-08T20:22:19.635Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-11-08T20:22:19.635Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-11-08T20:22:19.656Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-11-08T20:22:19.656Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-11-08T20:22:19.676Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-11-08T20:22:19.676Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-11-08T20:22:19.696Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-11-08T20:22:19.696Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-11-08T20:22:19.712Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-11-08T20:22:19.712Zdownloading input: /input/build/work/scripts/test_up.sh
222024-11-08T20:22:19.731Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-11-08T20:22:19.731Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-11-08T20:22:19.749Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-11-08T20:22:19.752Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-11-08T20:22:20.487Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-11-08T20:22:20.487Zstarting task 0: "setup"
282024-11-08T20:22:20.498Z++ uname -s
292024-11-08T20:22:20.501Z+ kern=SunOS
302024-11-08T20:22:20.501Z+ case "$kern" in
312024-11-08T20:22:20.501Z+ groupadd -g 12345 build
322024-11-08T20:22:20.504Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-11-08T20:22:22.513Z+ zfs create -o mountpoint=/work rpool/work
342024-11-08T20:22:23.251Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-11-08T20:22:23.253Z+ home_fs=zfs
362024-11-08T20:22:23.253Z+ [[ zfs == autofs ]]
372024-11-08T20:22:23.253Z+ mkdir -p /home/build
382024-11-08T20:22:23.256Z+ chown build:build /home/build /work
392024-11-08T20:22:25.258Z+ chmod 0700 /home/build /work
402024-11-08T20:22:25.260Zprocess exited: duration 4773 ms, exit code 0
 
412024-11-08T20:22:25.266Zstarting task 1: "authentication"
422024-11-08T20:22:25.283Zprocess exited: duration 17 ms, exit code 0
 
432024-11-08T20:22:25.289Zstarting task 2: "build"
442024-11-08T20:22:25.294Z+ banner cores
452024-11-08T20:22:25.323Z
462024-11-08T20:22:25.323Z #### #### ##### ###### ####
472024-11-08T20:22:25.323Z # # # # # # # #
482024-11-08T20:22:25.323Z # # # # # ##### ####
492024-11-08T20:22:25.323Z # # # ##### # #
502024-11-08T20:22:25.323Z # # # # # # # # #
512024-11-08T20:22:25.323Z #### #### # # ###### ####
522024-11-08T20:22:25.323Z
532024-11-08T20:22:25.326Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-11-08T20:22:25.334Z+ echo 'input bins dir contains:'
552024-11-08T20:22:25.334Zinput bins dir contains:
562024-11-08T20:22:25.336Z+ ls -ltr /input/build/work/bins
572024-11-08T20:22:25.336Ztotal 762383
582024-11-08T20:22:25.336Z-rw-r--r-- 1 root root 109277211 Nov 8 20:22 crucible-downstairs.gz
592024-11-08T20:22:25.336Z-rw-r--r-- 1 root root 104946520 Nov 8 20:22 crucible-hammer.gz
602024-11-08T20:22:25.336Z-rw-r--r-- 1 root root 109386824 Nov 8 20:22 crutest.gz
612024-11-08T20:22:25.336Z-rw-r--r-- 1 root root 66480353 Nov 8 20:22 dsc.gz
622024-11-08T20:22:25.336Z+ banner unpack
632024-11-08T20:22:25.339Z
642024-11-08T20:22:25.339Z # # # # ##### ## #### # #
652024-11-08T20:22:25.339Z # # ## # # # # # # # # #
662024-11-08T20:22:25.339Z # # # # # # # # # # ####
672024-11-08T20:22:25.339Z # # # # # ##### ###### # # #
682024-11-08T20:22:25.339Z # # # ## # # # # # # #
692024-11-08T20:22:25.339Z #### # # # # # #### # #
702024-11-08T20:22:25.339Z
712024-11-08T20:22:25.339Z+ mkdir -p /var/tmp/bins
722024-11-08T20:22:25.341Z+ for t in "$input/bins/"*.gz
732024-11-08T20:22:25.341Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-11-08T20:22:25.344Z+ b=crucible-downstairs.gz
752024-11-08T20:22:25.344Z+ b=crucible-downstairs
762024-11-08T20:22:25.344Z+ gunzip
772024-11-08T20:22:27.590Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-11-08T20:22:27.593Z+ for t in "$input/bins/"*.gz
792024-11-08T20:22:27.595Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-11-08T20:22:27.595Z+ b=crucible-hammer.gz
812024-11-08T20:22:27.595Z+ b=crucible-hammer
822024-11-08T20:22:27.595Z+ gunzip
832024-11-08T20:22:29.766Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-11-08T20:22:29.769Z+ for t in "$input/bins/"*.gz
852024-11-08T20:22:29.771Z++ basename /input/build/work/bins/crutest.gz
862024-11-08T20:22:29.771Z+ b=crutest.gz
872024-11-08T20:22:29.771Z+ b=crutest
882024-11-08T20:22:29.771Z+ gunzip
892024-11-08T20:22:32.042Z+ chmod +x /var/tmp/bins/crutest
902024-11-08T20:22:32.045Z+ for t in "$input/bins/"*.gz
912024-11-08T20:22:32.048Z++ basename /input/build/work/bins/dsc.gz
922024-11-08T20:22:32.048Z+ b=dsc.gz
932024-11-08T20:22:32.048Z+ b=dsc
942024-11-08T20:22:32.048Z+ gunzip
952024-11-08T20:22:33.402Z+ chmod +x /var/tmp/bins/dsc
962024-11-08T20:22:33.405Z+ export BINDIR=/var/tmp/bins
972024-11-08T20:22:33.405Z+ BINDIR=/var/tmp/bins
982024-11-08T20:22:33.405Z+ jobpid=1137
992024-11-08T20:22:33.407Z+ echo 'Setup debug logging'
1002024-11-08T20:22:33.407ZSetup debug logging
1012024-11-08T20:22:33.407Z+ mkdir /tmp/debug
1022024-11-08T20:22:33.407Z+ sleep 7200
1032024-11-08T20:22:33.407Z+ psrinfo -v
1042024-11-08T20:22:33.412Z+ df -h
1052024-11-08T20:22:33.438Z+ prstat -d d -mLc 1
1062024-11-08T20:22:33.441Z+ iostat -T d -xn 1
1072024-11-08T20:22:33.441Z+ mpstat -T d 1
1082024-11-08T20:22:33.441Z+ vmstat -T d -p 1
1092024-11-08T20:22:33.443Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102024-11-08T20:22:33.443Z+ banner test_up_unencrypted
1112024-11-08T20:22:33.443Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122024-11-08T20:22:33.444Z
1132024-11-08T20:22:33.444Z ##### ###### #### ##### # # ##### # # # #
1142024-11-08T20:22:33.444Z # # # # # # # # # # ## #
1152024-11-08T20:22:33.444Z # ##### #### # # # # # # # # # #
1162024-11-08T20:22:33.444Z # # # # # # ##### # # # # #
1172024-11-08T20:22:33.444Z # # # # # # # # # # # ##
1182024-11-08T20:22:33.444Z # ###### #### # ####### #### # ####### #### # #
1192024-11-08T20:22:33.444Z
1202024-11-08T20:22:33.447Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1212024-11-08T20:22:33.455Z/input/build/work
1222024-11-08T20:22:33.487ZTurn off color for downstairs dump
1232024-11-08T20:22:33.491Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1242024-11-08T20:22:33.491ZCreating 3 downstairs regions
1252024-11-08T20:22:33.843ZStarting 3 downstairs
1262024-11-08T20:22:33.846Zdsc started at PID: 1182
1272024-11-08T20:22:39.017ZDisable automatic restart on all downstairs
1282024-11-08T20:22:39.068Z
1292024-11-08T20:22:39.068ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1302024-11-08T20:22:39.072ZRunning test: span
1312024-11-08T20:22:43.253ZCompleted test: span
1322024-11-08T20:22:48.260ZRunning test: big
1332024-11-08T20:22:52.499ZCompleted test: big
1342024-11-08T20:22:57.506ZRunning test: dep
1352024-11-08T20:23:10.474ZCompleted test: dep
1362024-11-08T20:23:15.482ZRunning test: balloon
1372024-11-08T20:23:30.016ZCompleted test: balloon
1382024-11-08T20:23:35.024ZRunning test: deactivate
1392024-11-08T20:24:29.333ZCompleted test: deactivate
1402024-11-08T20:24:34.341ZRunning hammer
1412024-11-08T20:24:38.141ZRun repair tests
1422024-11-08T20:24:38.144Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1432024-11-08T20:24:38.204Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.920895652Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"task":"crutest"}
1442024-11-08T20:24:38.207Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.921510668Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"task":"crutest"}
1452024-11-08T20:24:38.207Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.923199756Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"task":"crutest"}
1462024-11-08T20:24:38.207Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.923362544Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219}
1472024-11-08T20:24:38.207Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.923402718Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219}
1482024-11-08T20:24:38.207Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.923417066Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219}
1492024-11-08T20:24:38.207Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.923431844Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219}
1502024-11-08T20:24:38.207Z{"msg":"Crucible c8ee2b41-acda-4816-906c-60edee2b44ad has session id: 10f5d8f5-8149-4b92-b1a3-0e60b6c045d7","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.923910688Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1512024-11-08T20:24:38.207Z{"msg":"Upstairs opts: Upstairs UUID: c8ee2b41-acda-4816-906c-60edee2b44ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.92396767Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1522024-11-08T20:24:38.207Z{"msg":"Crucible stats registered with UUID: c8ee2b41-acda-4816-906c-60edee2b44ad","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.924053838Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1532024-11-08T20:24:38.241Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.957960058Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"task":"crutest"}
1542024-11-08T20:24:38.241Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958034728Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219}
1552024-11-08T20:24:38.245Z{"msg":"c8ee2b41-acda-4816-906c-60edee2b44ad active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958084571Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1562024-11-08T20:24:38.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958133094Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"0","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1572024-11-08T20:24:38.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958187307Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"0","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1582024-11-08T20:24:38.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958226212Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"1","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1592024-11-08T20:24:38.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958289113Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"1","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1602024-11-08T20:24:38.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958344776Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"2","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1612024-11-08T20:24:38.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.95838136Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"2","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1622024-11-08T20:24:38.245Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958413386Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1632024-11-08T20:24:38.245Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958447101Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1642024-11-08T20:24:38.245Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958480177Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1652024-11-08T20:24:38.245Z{"msg":"ds_connection connected from Ok(127.0.0.1:48936)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958514722Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1662024-11-08T20:24:38.245Z{"msg":"ds_connection connected from Ok(127.0.0.1:59128)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958548608Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1672024-11-08T20:24:38.245Z{"msg":"ds_connection connected from Ok(127.0.0.1:51743)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.958738241Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1682024-11-08T20:24:38.245Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.95918657Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"1","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1692024-11-08T20:24:38.245Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.959239133Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"1","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1702024-11-08T20:24:38.245Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.959273208Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"2","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1712024-11-08T20:24:38.245Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.959311013Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"2","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1722024-11-08T20:24:38.245Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.959335399Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"0","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1732024-11-08T20:24:38.245Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.959369325Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"0","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1742024-11-08T20:24:38.245Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.959829501Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"1","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1752024-11-08T20:24:38.245Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.959871426Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"0","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1762024-11-08T20:24:38.245Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.959969852Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"2","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1772024-11-08T20:24:38.245Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.96027661Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"1","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1782024-11-08T20:24:38.245Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960317864Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"1","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1792024-11-08T20:24:38.246Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960333852Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1802024-11-08T20:24:38.246Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.96034665Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"0","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1812024-11-08T20:24:38.246Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960368517Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"0","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1822024-11-08T20:24:38.246Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960382675Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1832024-11-08T20:24:38.246Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960402123Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"2","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1842024-11-08T20:24:38.246Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960415901Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"2","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1852024-11-08T20:24:38.246Z{"msg":"[0]R flush_numbers: [1523, 1523, 1523, 1523, 1523]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960436878Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1862024-11-08T20:24:38.246Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960451016Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1872024-11-08T20:24:38.246Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960464554Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1882024-11-08T20:24:38.246Z{"msg":"[1]R flush_numbers: [1523, 1523, 1523, 1523, 1523]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960478682Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1892024-11-08T20:24:38.246Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960499689Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1902024-11-08T20:24:38.246Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960513647Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1912024-11-08T20:24:38.246Z{"msg":"[2]R flush_numbers: [1523, 1523, 1523, 1523, 1523]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960535304Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1922024-11-08T20:24:38.246Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960548862Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1932024-11-08T20:24:38.246Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1942024-11-08T20:24:38.246Z,"time":"2024-11-08T20:24:36.96056956Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1952024-11-08T20:24:38.246Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960589687Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1962024-11-08T20:24:38.246Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960610474Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1972024-11-08T20:24:38.246Z{"msg":"Next flush: 1524","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960623992Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1982024-11-08T20:24:38.246Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.96063703Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"":"downstairs","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
1992024-11-08T20:24:38.246Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960657218Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
2002024-11-08T20:24:38.246Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960669676Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"0","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
2012024-11-08T20:24:38.246Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960690263Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"1","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
2022024-11-08T20:24:38.246Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960703801Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"client":"2","":"downstairs","session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
2032024-11-08T20:24:38.246Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960717049Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
2042024-11-08T20:24:38.246Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960729878Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
2052024-11-08T20:24:38.246Z{"msg":"c8ee2b41-acda-4816-906c-60edee2b44ad is now active with session: 10f5d8f5-8149-4b92-b1a3-0e60b6c045d7","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960742956Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
2062024-11-08T20:24:38.246Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960763113Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219,"session_id":"10f5d8f5-8149-4b92-b1a3-0e60b6c045d7"}
2072024-11-08T20:24:38.246Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:36.960776011Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1219}
2082024-11-08T20:24:38.246ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2092024-11-08T20:24:38.246ZFill test
2102024-11-08T20:24:38.255ZRead and Verify all blocks (0..50 range:false)
2112024-11-08T20:24:38.261ZWrote out file "/var/tmp/test_up-build/verify_file"
2122024-11-08T20:24:38.261ZCLIENT: Tests done. All submitted work has been ACK'd
2132024-11-08T20:24:38.265Z----------------------------------------------------------------
2142024-11-08T20:24:38.265Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2152024-11-08T20:24:38.265ZStates: Active Active Active
2162024-11-08T20:24:38.265ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2172024-11-08T20:24:38.265Z 1002 Acked Read 50 Done Done Done false
2182024-11-08T20:24:38.265Z STATES DS:0 DS:1 DS:2 TOTAL
2192024-11-08T20:24:38.265Z Sent 0 0 0 0
2202024-11-08T20:24:38.265Z Done 1 1 1 3
2212024-11-08T20:24:38.265Z Skipped 0 0 0 0
2222024-11-08T20:24:38.266Z Error 0 0 0 0
2232024-11-08T20:24:38.266ZLast Flush: 1001 1001 1001
2242024-11-08T20:24:38.266ZDownstairs last five completed: 1001 1000
2252024-11-08T20:24:38.266ZUpstairs last five completed: 1002 1001 1000
2262024-11-08T20:24:38.266ZCLIENT: Up:0 ds:1 act:3
2272024-11-08T20:24:42.264Z----------------------------------------------------------------
2282024-11-08T20:24:42.264Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2292024-11-08T20:24:42.264ZDownstairs last five completed: 1003 1002 1001 1000
2302024-11-08T20:24:42.264ZUpstairs last five completed: 1003 1002 1001 1000
2312024-11-08T20:24:42.266ZCLIENT: Up:0 ds:0 act:3
2322024-11-08T20:24:42.266ZCLIENT: All crucible jobs finished, exiting program
2332024-11-08T20:24:42.275ZRepair setup passed
2342024-11-08T20:24:42.279ZCopy the region for /var/tmp/test_up-build/8830
2352024-11-08T20:24:42.282Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2362024-11-08T20:24:42.284Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2372024-11-08T20:24:42.342Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.058285277Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"task":"crutest"}
2382024-11-08T20:24:42.344Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.058978692Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"task":"crutest"}
2392024-11-08T20:24:42.345Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.060825648Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"task":"crutest"}
2402024-11-08T20:24:42.347Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.06103172Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225}
2412024-11-08T20:24:42.347Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.06110634Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225}
2422024-11-08T20:24:42.348Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.061158093Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225}
2432024-11-08T20:24:42.348Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.061190728Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225}
2442024-11-08T20:24:42.348Z{"msg":"Crucible 3b038207-a030-48bf-b46e-ec45ee0f57e0 has session id: 316c3c44-47fd-4331-b987-3bea2dd79798","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.061627288Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2452024-11-08T20:24:42.348Z{"msg":"Upstairs opts: Upstairs UUID: 3b038207-a030-48bf-b46e-ec45ee0f57e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.061688529Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2462024-11-08T20:24:42.348Z{"msg":"Crucible stats registered with UUID: 3b038207-a030-48bf-b46e-ec45ee0f57e0","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.061709957Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2472024-11-08T20:24:42.380Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.095893348Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"task":"crutest"}
2482024-11-08T20:24:42.380Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.095957659Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225}
2492024-11-08T20:24:42.380Z{"msg":"3b038207-a030-48bf-b46e-ec45ee0f57e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.095992755Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2502024-11-08T20:24:42.381Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096037179Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"0","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2512024-11-08T20:24:42.384Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096085612Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"0","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2522024-11-08T20:24:42.384Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096142214Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"1","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2532024-11-08T20:24:42.384Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096182299Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"1","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2542024-11-08T20:24:42.384Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096205836Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"2","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2552024-11-08T20:24:42.384Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096230682Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"2","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2562024-11-08T20:24:42.384Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096529771Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2572024-11-08T20:24:42.384Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096594742Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2582024-11-08T20:24:42.384Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096631517Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2592024-11-08T20:24:42.384Z{"msg":"ds_connection connected from Ok(127.0.0.1:41203)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096677871Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2602024-11-08T20:24:42.384Z{"msg":"ds_connection connected from Ok(127.0.0.1:39314)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096720175Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2612024-11-08T20:24:42.384Z{"msg":"ds_connection connected from Ok(127.0.0.1:35135)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.096761769Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2622024-11-08T20:24:42.384Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.097202368Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"2","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2632024-11-08T20:24:42.384Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.097247342Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"2","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2642024-11-08T20:24:42.384Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.097274798Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"1","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2652024-11-08T20:24:42.384Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.097301185Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"1","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2662024-11-08T20:24:42.384Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.09733873Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"0","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2672024-11-08T20:24:42.384Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.097401801Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"0","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2682024-11-08T20:24:42.384Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.097925299Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"1","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2692024-11-08T20:24:42.384Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.097984591Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"0","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2702024-11-08T20:24:42.384Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098019146Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"2","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2712024-11-08T20:24:42.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098335533Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"1","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2722024-11-08T20:24:42.384Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098388856Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"1","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2732024-11-08T20:24:42.384Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098441638Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2742024-11-08T20:24:42.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098484372Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"0","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2752024-11-08T20:24:42.384Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098526117Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"0","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2762024-11-08T20:24:42.384Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098563501Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2772024-11-08T20:24:42.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098596757Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"2","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2782024-11-08T20:24:42.384Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098633222Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"2","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2792024-11-08T20:24:42.384Z{"msg":"[0]R flush_numbers: [1524, 1524, 1524, 1524, 1524]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098678956Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2802024-11-08T20:24:42.384Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.09871884Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2812024-11-08T20:24:42.384ZWait for a query_work_queue command to finish before sending IO
2822024-11-08T20:24:42.384Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098765494Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2832024-11-08T20:24:42.384Z{"msg":"[1]R flush_numbers: [1524, 1524, 1524, 1524, 1524]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098805448Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2842024-11-08T20:24:42.384Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098846902Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2852024-11-08T20:24:42.385Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098887017Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2862024-11-08T20:24:42.385Z{"msg":"[2]R flush_numbers: [1524, 1524, 1524, 1524, 1524]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098930341Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2872024-11-08T20:24:42.385Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0Region: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2882024-11-08T20:24:42.385Z,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.098981324Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2892024-11-08T20:24:42.385Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099022738Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2902024-11-08T20:24:42.385Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099048405Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2912024-11-08T20:24:42.385Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099063443Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2922024-11-08T20:24:42.385Z{"msg":"Next flush: 1525","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.09908501Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2932024-11-08T20:24:42.385Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099099718Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"":"downstairs","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2942024-11-08T20:24:42.385Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099113286Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2952024-11-08T20:24:42.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099132813Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"0","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2962024-11-08T20:24:42.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099146631Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"1","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2972024-11-08T20:24:42.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099166749Z","hostname":"Loading write count information from file "/var/tmp/test_up-build/verify_file"
2982024-11-08T20:24:42.385Zw-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"client":"2","":"downstairs","session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
2992024-11-08T20:24:42.385Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099185706Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
3002024-11-08T20:24:42.385Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099205273Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"Fill test
3012024-11-08T20:24:42.385Z316c3c44-47fd-4331-b987-3bea2dd79798"}
3022024-11-08T20:24:42.385Z{"msg":"3b038207-a030-48bf-b46e-ec45ee0f57e0 is now active with session: 316c3c44-47fd-4331-b987-3bea2dd79798","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099221331Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
3032024-11-08T20:24:42.385Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099241358Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225,"session_id":"316c3c44-47fd-4331-b987-3bea2dd79798"}
3042024-11-08T20:24:42.385Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:41.099254157Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225}
3052024-11-08T20:24:42.397ZRead and Verify all blocks (0..50 range:false)
3062024-11-08T20:24:42.403ZWrote out file "/var/tmp/test_up-build/verify_file"
3072024-11-08T20:24:42.403ZCLIENT: Tests done. All submitted work has been ACK'd
3082024-11-08T20:24:42.406Z----------------------------------------------------------------
3092024-11-08T20:24:42.406Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3102024-11-08T20:24:42.406ZStates: Active Active Active
3112024-11-08T20:24:42.406ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3122024-11-08T20:24:42.406Z 1002 Acked Read 50 Done Done Done false
3132024-11-08T20:24:42.406Z STATES DS:0 DS:1 DS:2 TOTAL
3142024-11-08T20:24:42.406Z Sent 0 0 0 0
3152024-11-08T20:24:42.406Z Done 1 1 1 3
3162024-11-08T20:24:42.406Z Skipped 0 0 0 0
3172024-11-08T20:24:42.406Z Error 0 0 0 0
3182024-11-08T20:24:42.406ZLast Flush: 1001 1001 1001
3192024-11-08T20:24:42.406ZDownstairs last five completed: 1001 1000
3202024-11-08T20:24:42.406ZUpstairs last five completed: 1002 1001 1000
3212024-11-08T20:24:42.406ZCLIENT: Up:0 ds:1 act:3
3222024-11-08T20:24:46.406Z----------------------------------------------------------------
3232024-11-08T20:24:46.406Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3242024-11-08T20:24:46.406ZDownstairs last five completed: 1003 1002 1001 1000
3252024-11-08T20:24:46.406ZUpstairs last five completed: 1003 1002 1001 1000
3262024-11-08T20:24:46.406ZCLIENT: Up:0 ds:0 act:3
3272024-11-08T20:24:46.406ZCLIENT: All crucible jobs finished, exiting program
3282024-11-08T20:24:46.409Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:45.121560829Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1225}
3292024-11-08T20:24:46.413ZRepair part 1 passed
3302024-11-08T20:24:46.416Z
3312024-11-08T20:24:46.416ZKill the current downstairs
3322024-11-08T20:24:46.578ZDownstairs 2 stopped
3332024-11-08T20:24:46.578Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3342024-11-08T20:24:46.580ZNow put back the original so we have a mismatch
3352024-11-08T20:24:46.580Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3362024-11-08T20:24:46.583ZRestart downstairs with old directory
3372024-11-08T20:24:46.635Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3382024-11-08T20:24:46.651ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3392024-11-08T20:24:46.655Z 0 000-009 32 32 31 1525 1525 1524 F F F <---
3402024-11-08T20:24:46.655Z 1 010-019 32 32 31 1525 1525 1524 F F F <---
3412024-11-08T20:24:46.655Z 2 020-029 32 32 31 1525 1525 1524 F F F <---
3422024-11-08T20:24:46.655Z 3 030-039 32 32 31 1525 1525 1524 F F F <---
3432024-11-08T20:24:46.655Z 4 040-049 32 32 31 1525 1525 1524 F F F <---
3442024-11-08T20:24:46.655ZMax gen: 32, Max flush: 1525
3452024-11-08T20:24:46.655ZError: Difference in extent metadata found!
3462024-11-08T20:24:46.657Zdump test found error as expected
3472024-11-08T20:24:46.658Z
3482024-11-08T20:24:46.658Z
3492024-11-08T20:24:46.658Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3502024-11-08T20:24:46.715Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.430129309Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"task":"crutest"}
3512024-11-08T20:24:46.717Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.430669714Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"task":"crutest"}
3522024-11-08T20:24:46.717Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.432406525Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"task":"crutest"}
3532024-11-08T20:24:46.717Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.43259362Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233}
3542024-11-08T20:24:46.717Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"0fd98d0d5d909611f8b408ccda2559d35fe117bb\",\n git_commit_timestamp: \"2024-11-08T20:05:44.000000000Z\",\n git_branch: \"mkeeter/drop-skipped-backpressure-guard\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.432640003Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233}
3552024-11-08T20:24:46.717Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.4326664Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233}
3562024-11-08T20:24:46.717Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.432686857Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233}
3572024-11-08T20:24:46.720Z{"msg":"Crucible 18206a41-9814-4edb-bfa3-2d0efa04a825 has session id: 65b9455b-4a00-4963-935b-1a18a3ceb0aa","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.433073354Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3582024-11-08T20:24:46.720Z{"msg":"Upstairs opts: Upstairs UUID: 18206a41-9814-4edb-bfa3-2d0efa04a825, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.433150913Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3592024-11-08T20:24:46.720Z{"msg":"Crucible stats registered with UUID: 18206a41-9814-4edb-bfa3-2d0efa04a825","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.433193947Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3602024-11-08T20:24:46.752Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.467687356Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"task":"crutest"}
3612024-11-08T20:24:46.752Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.467772255Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233}
3622024-11-08T20:24:46.752Z{"msg":"18206a41-9814-4edb-bfa3-2d0efa04a825 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.467794722Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3632024-11-08T20:24:46.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.467817509Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3642024-11-08T20:24:46.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.467873811Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3652024-11-08T20:24:46.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.467960429Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3662024-11-08T20:24:46.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.46802606Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3672024-11-08T20:24:46.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.468054206Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3682024-11-08T20:24:46.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.468090081Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3692024-11-08T20:24:46.755Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.468124936Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3702024-11-08T20:24:46.755Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.468159711Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3712024-11-08T20:24:46.755Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.468196017Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3722024-11-08T20:24:46.755Z{"msg":"ds_connection connected from Ok(127.0.0.1:60474)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.468344896Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3732024-11-08T20:24:46.755Z{"msg":"ds_connection connected from Ok(127.0.0.1:44087)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.468378312Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3742024-11-08T20:24:46.755Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:45.468428865Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3752024-11-08T20:24:46.755Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:45.46846157Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3762024-11-08T20:24:46.755Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:45.46853328Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3772024-11-08T20:24:46.755Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.468634036Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3782024-11-08T20:24:46.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.468995437Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3792024-11-08T20:24:46.755Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.469032042Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3802024-11-08T20:24:46.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.469059508Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3812024-11-08T20:24:46.755Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.469090164Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3822024-11-08T20:24:46.755Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.46955333Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3832024-11-08T20:24:46.755Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.469667154Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3842024-11-08T20:24:46.755Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.469857478Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3852024-11-08T20:24:46.755Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.469902172Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3862024-11-08T20:24:46.755Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.469928608Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3872024-11-08T20:24:46.756Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.469956475Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3882024-11-08T20:24:46.756Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.469972042Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3892024-11-08T20:24:46.756Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:45.469993479Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3902024-11-08T20:24:56.756Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.469840711Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3912024-11-08T20:24:56.761Z{"msg":"ds_connection connected from Ok(127.0.0.1:62825)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.470128811Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3922024-11-08T20:24:56.761Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.471018359Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3932024-11-08T20:24:56.761Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.471059494Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3942024-11-08T20:24:56.761Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.471557465Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3952024-11-08T20:24:56.761Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.471906327Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3962024-11-08T20:24:56.761Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.471936743Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3972024-11-08T20:24:56.761Z{"msg":"[0]R flush_numbers: [1525, 1525, 1525, 1525, 1525]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.471953011Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3982024-11-08T20:24:56.761Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.471974598Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
3992024-11-08T20:24:56.761Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.471988826Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4002024-11-08T20:24:56.761Z{"msg":"[1]R flush_numbers: [1525, 1525, 1525, 1525, 1525]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472002614Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4012024-11-08T20:24:56.761Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472016242Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4022024-11-08T20:24:56.761Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472037019Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4032024-11-08T20:24:56.761Z{"msg":"[2]R flush_numbers: [1524, 1524, 1524, 1524, 1524]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472050987Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4042024-11-08T20:24:56.762Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472071264Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4052024-11-08T20:24:56.762Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472084823Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4062024-11-08T20:24:56.762Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.47210479Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4072024-11-08T20:24:56.762Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472118858Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4082024-11-08T20:24:56.762Z{"msg":"Next flush: 1526","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472138735Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4092024-11-08T20:24:56.762Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472152073Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4102024-11-08T20:24:56.762Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472173241Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4112024-11-08T20:24:56.762Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472188498Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4122024-11-08T20:24:56.762Z{"msg":"extent:0 flush: 1525 1525 1524 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472210935Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4132024-11-08T20:24:56.762Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472225763Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4142024-11-08T20:24:56.762Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472240161Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4152024-11-08T20:24:56.762Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472254539Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4162024-11-08T20:24:56.762Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472275596Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4172024-11-08T20:24:56.762Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472290174Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4182024-11-08T20:24:56.762Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472317371Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4192024-11-08T20:24:56.762Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472340307Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4202024-11-08T20:24:56.762Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472355205Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4212024-11-08T20:24:56.762Z{"msg":"extent:1 flush: 1525 1525 1524 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472376733Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4222024-11-08T20:24:56.762Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472391551Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4232024-11-08T20:24:56.762Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472412737Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4242024-11-08T20:24:56.762Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472427566Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4252024-11-08T20:24:56.762Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472448593Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4262024-11-08T20:24:56.762Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472463181Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4272024-11-08T20:24:56.762Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472484488Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4282024-11-08T20:24:56.762Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472498716Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4292024-11-08T20:24:56.762Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472513504Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4302024-11-08T20:24:56.762Z{"msg":"extent:2 flush: 1525 1525 1524 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472528032Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4312024-11-08T20:24:56.762Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472549479Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4322024-11-08T20:24:56.762Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472564327Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4332024-11-08T20:24:56.762Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472585894Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4342024-11-08T20:24:56.762Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472600752Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4352024-11-08T20:24:56.762Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472621839Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4362024-11-08T20:24:56.762Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472636807Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4372024-11-08T20:24:56.762Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472657184Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4382024-11-08T20:24:56.762Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472671832Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4392024-11-08T20:24:56.762Z{"msg":"extent:3 flush: 1525 1525 1524 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472693199Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4402024-11-08T20:24:56.762Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472707887Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4412024-11-08T20:24:56.763Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472728754Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4422024-11-08T20:24:56.763Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472743642Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4432024-11-08T20:24:56.763Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472764579Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4442024-11-08T20:24:56.763Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472779287Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4452024-11-08T20:24:56.763Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472794075Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4462024-11-08T20:24:56.763Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472807373Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4472024-11-08T20:24:56.763Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472821831Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4482024-11-08T20:24:56.763Z{"msg":"extent:4 flush: 1525 1525 1524 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472841489Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4492024-11-08T20:24:56.763Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472868575Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4502024-11-08T20:24:56.763Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472891262Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4512024-11-08T20:24:56.763Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.47290683Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4522024-11-08T20:24:56.763Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472932316Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4532024-11-08T20:24:56.763Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472947134Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4542024-11-08T20:24:56.763Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472969211Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4552024-11-08T20:24:56.763Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.472983179Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4562024-11-08T20:24:56.763Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.473003946Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4572024-11-08T20:24:56.763Z{"msg":"starting reconciliation 3191babb-9261-48dc-9aff-41a1d16638f2: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.473018824Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4582024-11-08T20:24:56.763Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.473043341Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4592024-11-08T20:24:56.766Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1526, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1526, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1526, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1526, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1526, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.473116941Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4602024-11-08T20:24:56.766Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.473132729Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4612024-11-08T20:24:56.766Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.473152066Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4622024-11-08T20:24:56.766Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.473359707Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4632024-11-08T20:24:56.766Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.473616262Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4642024-11-08T20:24:56.766Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.473656077Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4652024-11-08T20:24:56.773Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.487265686Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4662024-11-08T20:24:56.775Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.48767663Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4672024-11-08T20:24:56.775Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.488001475Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4682024-11-08T20:24:56.775Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.488285626Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4692024-11-08T20:24:56.775Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.488340588Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4702024-11-08T20:24:56.783Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.497027354Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4712024-11-08T20:24:56.786Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.497562771Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4722024-11-08T20:24:56.786Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.497836953Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4732024-11-08T20:24:56.786Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.498129093Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4742024-11-08T20:24:56.786Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.498178856Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4752024-11-08T20:24:56.793Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.50724668Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4762024-11-08T20:24:56.795Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.507873474Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4772024-11-08T20:24:56.795Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.508270539Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4782024-11-08T20:24:56.795Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.50848155Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4792024-11-08T20:24:56.795Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.508533123Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4802024-11-08T20:24:56.804Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.517844053Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4812024-11-08T20:24:56.806Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.518386399Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4822024-11-08T20:24:56.806Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.518659341Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4832024-11-08T20:24:56.806Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.518967909Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4842024-11-08T20:24:56.806Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.519038639Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4852024-11-08T20:24:56.813Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.527343548Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4862024-11-08T20:24:56.815Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.527660914Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4872024-11-08T20:24:56.816Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.527706188Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"0","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4882024-11-08T20:24:56.816Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.527726335Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"1","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4892024-11-08T20:24:56.816Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.52776049Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"client":"2","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4902024-11-08T20:24:56.816Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
4912024-11-08T20:24:56.816Z,"time":"2024-11-08T20:24:55.527793716Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4922024-11-08T20:24:56.816Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.527827941Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4932024-11-08T20:24:56.816Z{"msg":"18206a41-9814-4edb-bfa3-2d0efa04a825 is now active with session: 65b9455b-4a00-4963-935b-1a18a3ceb0aa","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.527871945Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4942024-11-08T20:24:56.816Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-11-08T20:24:55.527917339Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233}
4952024-11-08T20:24:56.816Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-11-08T20:24:55.527945105Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233,"reconcile":"3191babb-9261-48dc-9aff-41a1d16638f2","":"downstairs","":"downstairs","session_id":"65b9455b-4a00-4963-935b-1a18a3ceb0aa"}
4962024-11-08T20:24:56.816ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4972024-11-08T20:24:56.816ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4982024-11-08T20:24:56.816ZRead and Verify all blocks (0..50 range:true)
4992024-11-08T20:24:56.820ZWrote out file "/var/tmp/test_up-build/verify_file"
5002024-11-08T20:24:56.820ZVerify test completed
5012024-11-08T20:24:56.823ZWrote out file "/var/tmp/test_up-build/verify_file"
5022024-11-08T20:24:56.823ZCLIENT: Tests done. All submitted work has been ACK'd
5032024-11-08T20:24:56.823Z----------------------------------------------------------------
5042024-11-08T20:24:56.823Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5052024-11-08T20:24:56.823ZStates: Active Active Active
5062024-11-08T20:24:56.823ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5072024-11-08T20:24:56.823Z 1000 Acked Read 50 Done Done Done false
5082024-11-08T20:24:56.823Z STATES DS:0 DS:1 DS:2 TOTAL
5092024-11-08T20:24:56.823Z Sent 0 0 0 0
5102024-11-08T20:24:56.823Z Done 1 1 1 3
5112024-11-08T20:24:56.823Z Skipped 0 0 0 0
5122024-11-08T20:24:56.823Z Error 0 0 0 0
5132024-11-08T20:24:56.823ZLast Flush: 0 0 0
5142024-11-08T20:24:56.823ZDownstairs last five completed:
5152024-11-08T20:24:56.823ZUpstairs last five completed: 1000
5162024-11-08T20:24:56.823ZCLIENT: Up:0 ds:1 act:3
5172024-11-08T20:25:00.823Z----------------------------------------------------------------
5182024-11-08T20:25:00.823Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5192024-11-08T20:25:00.823ZDownstairs last five completed: 1001 1000
5202024-11-08T20:25:00.823ZUpstairs last five completed: 1001 1000
5212024-11-08T20:25:00.823ZCLIENT: Up:0 ds:0 act:3
5222024-11-08T20:25:00.823ZCLIENT: All crucible jobs finished, exiting program
5232024-11-08T20:25:00.826Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-08T20:24:59.536534463Z","hostname":"w-01JC6Q02X5E25PDH9N1636SCK7","pid":1233}
5242024-11-08T20:25:00.830ZRepair part 2 passed
5252024-11-08T20:25:00.833Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5262024-11-08T20:25:00.848ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5272024-11-08T20:25:00.848Z 0 000-009 32 32 32 1525 1525 1525 F F F
5282024-11-08T20:25:00.850Z 1 010-019 32 32 32 1525 1525 1525 F F F
5292024-11-08T20:25:00.850Z 2 020-029 32 32 32 1525 1525 1525 F F F
5302024-11-08T20:25:00.850Z 3 030-039 32 32 32 1525 1525 1525 F F F
5312024-11-08T20:25:00.850Z 4 040-049 32 32 32 1525 1525 1525 F F F
5322024-11-08T20:25:00.850ZMax gen: 32, Max flush: 1525
5332024-11-08T20:25:00.852Zdump test passed
5342024-11-08T20:25:00.852Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5352024-11-08T20:25:00.867Z Extent 1
5362024-11-08T20:25:00.867ZGEN 32 32 32
5372024-11-08T20:25:00.867ZFLUSH_ID 1525 1525 1525
5382024-11-08T20:25:00.867ZDIRTY
5392024-11-08T20:25:00.867Z
5402024-11-08T20:25:00.868ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5412024-11-08T20:25:00.870Z010 A A A A A A
5422024-11-08T20:25:00.870Z011 A A A A A A
5432024-11-08T20:25:00.872Z012 A A A A A A
5442024-11-08T20:25:00.874Z013 A A A A A A
5452024-11-08T20:25:00.874Z014 A A A A A A
5462024-11-08T20:25:00.877Z015 A A A A A A
5472024-11-08T20:25:00.877Z016 A A A A A A
5482024-11-08T20:25:00.879Z017 A A A A A A
5492024-11-08T20:25:00.879Z018 A A A A A A
5502024-11-08T20:25:00.882Z019 A A A A A A
5512024-11-08T20:25:00.884Zdump extent test passed
5522024-11-08T20:25:00.884Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5532024-11-08T20:25:00.898ZExtent 2 Block in extent 0 Actual block 20
5542024-11-08T20:25:00.898Z
5552024-11-08T20:25:00.901Z DATA SHA256 VER
5562024-11-08T20:25:00.901Z------ ---------------------------------------------------------------- ---
5572024-11-08T20:25:00.901Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5582024-11-08T20:25:00.901Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5592024-11-08T20:25:00.904Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5602024-11-08T20:25:00.904Z
5612024-11-08T20:25:00.904ZNONCES 0 1 2 DIFF
5622024-11-08T20:25:00.904Z------ ------------------------ ------------------------ ------------------------ -----
5632024-11-08T20:25:00.904Z
5642024-11-08T20:25:00.904Z TAGS 0 1 2 DIFF
5652024-11-08T20:25:00.904Z------ -------------------------------- -------------------------------- -------------------------------- -----
5662024-11-08T20:25:00.904Z
5672024-11-08T20:25:00.904ZHASHES 0 1 2 DIFF
5682024-11-08T20:25:00.904Z------ ---------------- ---------------- ---------------- -----
5692024-11-08T20:25:00.904Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5702024-11-08T20:25:00.904Z
5712024-11-08T20:25:00.907Zdump block test passed
5722024-11-08T20:25:00.907ZInitial upstairs tests have completed, stopping all downstairs
5732024-11-08T20:25:05.967ZCreating 4 larger downstairs regions
5742024-11-08T20:25:06.514ZStarting 4 downstairs
5752024-11-08T20:25:11.521Zdsc restarted at PID: 1246
5762024-11-08T20:25:11.743ZNow do the replace-reconcile test
5772024-11-08T20:25:11.797ZUsing 8840 for the replacement port
5782024-11-08T20:27:41.301ZCompleted test: replace-reconcile
5792024-11-08T20:27:41.304ZNow do the replace-before-active test
5802024-11-08T20:30:17.884ZCompleted test: replace-before-active
5812024-11-08T20:30:17.886ZAll tests have completed, stopping all downstairs
5822024-11-08T20:30:17.941Z
5832024-11-08T20:30:17.943ZAll Tests have passed
5842024-11-08T20:30:17.943Z7:44 Test duration
5852024-11-08T20:30:17.943Z
5862024-11-08T20:30:17.943Zreal 7:44.422425800
5872024-11-08T20:30:17.943Zuser 1:03.879193195
5882024-11-08T20:30:17.943Zsys 12.158354394
5892024-11-08T20:30:17.944Ztrap 0.156375102
5902024-11-08T20:30:17.944Ztflt 0.016521493
5912024-11-08T20:30:17.944Zdflt 0.052354285
5922024-11-08T20:30:17.944Zkflt 0.001301059
5932024-11-08T20:30:17.944Zlock 5:51:09.959947595
5942024-11-08T20:30:17.944Zslp 24:58.363833369
5952024-11-08T20:30:17.944Zlat 9.599623604
5962024-11-08T20:30:17.944Zstop 0.120920140
5972024-11-08T20:30:17.944Z+ echo 'test-up-unencrypted ends'
5982024-11-08T20:30:17.944Ztest-up-unencrypted ends
5992024-11-08T20:30:22.949Zprocess exited: duration 472580 ms, exit code 0
6002024-11-08T20:30:22.949Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6012024-11-08T20:31:22.966Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6022024-11-08T20:31:22.967Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6032024-11-08T20:31:22.996Zfound 14 output files
6042024-11-08T20:31:22.997Zuploading: /tmp/test_up-build/dsc-out.txt (37140 bytes)
6052024-11-08T20:31:24.021Zuploaded: /tmp/test_up-build/dsc-out.txt
6062024-11-08T20:31:24.030Zuploading: /tmp/test_up-build/test_up_out.txt (10917610 bytes)
6072024-11-08T20:31:25.201Zuploaded: /tmp/test_up-build/test_up_out.txt
6082024-11-08T20:31:25.204Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (253726 bytes)
6092024-11-08T20:31:26.246Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6102024-11-08T20:31:26.246Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144659 bytes)
6112024-11-08T20:31:27.322Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6122024-11-08T20:31:27.322Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251120 bytes)
6132024-11-08T20:31:28.347Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6142024-11-08T20:31:28.350Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139452 bytes)
6152024-11-08T20:31:29.366Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6162024-11-08T20:31:29.366Zuploading: /tmp/debug/df.txt (1270 bytes)
6172024-11-08T20:31:30.375Zuploaded: /tmp/debug/df.txt
6182024-11-08T20:31:30.375Zuploading: /tmp/debug/dtrace.txt (423167 bytes)
6192024-11-08T20:31:31.406Zuploaded: /tmp/debug/dtrace.txt
6202024-11-08T20:31:31.406Zuploading: /tmp/debug/iostat.txt (159614 bytes)
6212024-11-08T20:31:31.422Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 159614 -> 162022
6222024-11-08T20:31:32.428Zuploaded: /tmp/debug/iostat.txt
6232024-11-08T20:31:32.428Zuploading: /tmp/debug/mpstat.txt (386904 bytes)
6242024-11-08T20:31:32.453Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 386904 -> 393474
6252024-11-08T20:31:33.459Zuploaded: /tmp/debug/mpstat.txt
6262024-11-08T20:31:33.459Zuploading: /tmp/debug/paging.txt (69854 bytes)
6272024-11-08T20:31:33.470Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 69854 -> 71172
6282024-11-08T20:31:34.477Zuploaded: /tmp/debug/paging.txt
6292024-11-08T20:31:34.477Zuploading: /tmp/debug/prstat.txt (729197 bytes)
6302024-11-08T20:31:34.513Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 729197 -> 744734
6312024-11-08T20:31:35.519Zuploaded: /tmp/debug/prstat.txt
6322024-11-08T20:31:35.519Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6332024-11-08T20:31:36.528Zuploaded: /tmp/debug/psrinfo.txt
6342024-11-08T20:31:36.528Zuploading: /tmp/debug/upstairs-info.txt (71258 bytes)
6352024-11-08T20:31:37.543Zuploaded: /tmp/debug/upstairs-info.txt