01JENYDE80YXY1KANJ05YBHS0S: test-up-unencrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JENYDTDGTRADCBFKR0QZ7FVD

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-12-09T15:06:25.001Zjob dependencies complete; ready to run (waiting for 15 m 27 s)
22024-12-09T15:07:23.399Zjob assigned to worker 01JENZA9QXPWE9QFN5XVGTSRPW [factory aws, i-02795827ac48330ba] (queued for 58 s)
32024-12-09T15:07:30.568Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-12-09T15:07:31.731Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-12-09T15:07:31.731Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-12-09T15:07:32.823Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-12-09T15:07:32.823Zdownloading input: /input/build/work/bins/crutest.gz
82024-12-09T15:07:33.932Zdownloaded input: /input/build/work/bins/crutest.gz
92024-12-09T15:07:33.933Zdownloading input: /input/build/work/bins/dsc.gz
102024-12-09T15:07:34.604Zdownloaded input: /input/build/work/bins/dsc.gz
112024-12-09T15:07:34.608Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-12-09T15:07:34.626Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-12-09T15:07:34.626Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-12-09T15:07:34.649Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-12-09T15:07:34.649Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-12-09T15:07:34.667Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-12-09T15:07:34.667Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-12-09T15:07:34.686Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-12-09T15:07:34.686Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-12-09T15:07:34.709Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-12-09T15:07:34.710Zdownloading input: /input/build/work/scripts/test_up.sh
222024-12-09T15:07:34.728Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-12-09T15:07:34.728Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-12-09T15:07:34.747Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-12-09T15:07:34.749Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-12-09T15:07:35.470Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-12-09T15:07:35.470Zstarting task 0: "setup"
282024-12-09T15:07:35.479Z++ uname -s
292024-12-09T15:07:35.482Z+ kern=SunOS
302024-12-09T15:07:35.482Z+ case "$kern" in
312024-12-09T15:07:35.482Z+ groupadd -g 12345 build
322024-12-09T15:07:35.484Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-12-09T15:07:37.491Z+ zfs create -o mountpoint=/work rpool/work
342024-12-09T15:07:37.648Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-12-09T15:07:37.651Z+ home_fs=zfs
362024-12-09T15:07:37.651Z+ [[ zfs == autofs ]]
372024-12-09T15:07:37.651Z+ mkdir -p /home/build
382024-12-09T15:07:37.653Z+ chown build:build /home/build /work
392024-12-09T15:07:39.655Z+ chmod 0700 /home/build /work
402024-12-09T15:07:39.703Zprocess exited: duration 4187 ms, exit code 0
 
412024-12-09T15:07:39.708Zstarting task 1: "authentication"
422024-12-09T15:07:39.725Zprocess exited: duration 16 ms, exit code 0
 
432024-12-09T15:07:39.730Zstarting task 2: "build"
442024-12-09T15:07:39.735Z+ banner cores
452024-12-09T15:07:39.738Z
462024-12-09T15:07:39.738Z #### #### ##### ###### ####
472024-12-09T15:07:39.738Z # # # # # # # #
482024-12-09T15:07:39.738Z # # # # # ##### ####
492024-12-09T15:07:39.738Z # # # ##### # #
502024-12-09T15:07:39.738Z # # # # # # # # #
512024-12-09T15:07:39.738Z #### #### # # ###### ####
522024-12-09T15:07:39.738Z
532024-12-09T15:07:39.740Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-12-09T15:07:39.747Z+ echo 'input bins dir contains:'
552024-12-09T15:07:39.747Zinput bins dir contains:
562024-12-09T15:07:39.749Z+ ls -ltr /input/build/work/bins
572024-12-09T15:07:39.749Ztotal 762797
582024-12-09T15:07:39.749Z-rw-r--r-- 1 root root 109276547 Dec 9 15:07 crucible-downstairs.gz
592024-12-09T15:07:39.749Z-rw-r--r-- 1 root root 105056395 Dec 9 15:07 crucible-hammer.gz
602024-12-09T15:07:39.749Z-rw-r--r-- 1 root root 109488947 Dec 9 15:07 crutest.gz
612024-12-09T15:07:39.749Z-rw-r--r-- 1 root root 66480149 Dec 9 15:07 dsc.gz
622024-12-09T15:07:39.749Z+ banner unpack
632024-12-09T15:07:39.752Z
642024-12-09T15:07:39.752Z # # # # ##### ## #### # #
652024-12-09T15:07:39.752Z # # ## # # # # # # # # #
662024-12-09T15:07:39.752Z # # # # # # # # # # ####
672024-12-09T15:07:39.752Z # # # # # ##### ###### # # #
682024-12-09T15:07:39.752Z # # # ## # # # # # # #
692024-12-09T15:07:39.752Z #### # # # # # #### # #
702024-12-09T15:07:39.752Z
712024-12-09T15:07:39.752Z+ mkdir -p /var/tmp/bins
722024-12-09T15:07:39.755Z+ for t in "$input/bins/"*.gz
732024-12-09T15:07:39.755Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-12-09T15:07:39.755Z+ b=crucible-downstairs.gz
752024-12-09T15:07:39.755Z+ b=crucible-downstairs
762024-12-09T15:07:39.755Z+ gunzip
772024-12-09T15:07:42.006Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-12-09T15:07:42.008Z+ for t in "$input/bins/"*.gz
792024-12-09T15:07:42.011Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-12-09T15:07:42.011Z+ b=crucible-hammer.gz
812024-12-09T15:07:42.011Z+ b=crucible-hammer
822024-12-09T15:07:42.011Z+ gunzip
832024-12-09T15:07:44.153Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-12-09T15:07:44.156Z+ for t in "$input/bins/"*.gz
852024-12-09T15:07:44.159Z++ basename /input/build/work/bins/crutest.gz
862024-12-09T15:07:44.159Z+ b=crutest.gz
872024-12-09T15:07:44.159Z+ b=crutest
882024-12-09T15:07:44.159Z+ gunzip
892024-12-09T15:07:46.463Z+ chmod +x /var/tmp/bins/crutest
902024-12-09T15:07:46.465Z+ for t in "$input/bins/"*.gz
912024-12-09T15:07:46.468Z++ basename /input/build/work/bins/dsc.gz
922024-12-09T15:07:46.470Z+ b=dsc.gz
932024-12-09T15:07:46.470Z+ b=dsc
942024-12-09T15:07:46.471Z+ gunzip
952024-12-09T15:07:47.842Z+ chmod +x /var/tmp/bins/dsc
962024-12-09T15:07:47.844Z+ export BINDIR=/var/tmp/bins
972024-12-09T15:07:47.844Z+ BINDIR=/var/tmp/bins
982024-12-09T15:07:47.844Z+ jobpid=1141
992024-12-09T15:07:47.848Z+ echo 'Setup debug logging'
1002024-12-09T15:07:47.848ZSetup debug logging
1012024-12-09T15:07:47.848Z+ mkdir /tmp/debug
1022024-12-09T15:07:47.848Z+ sleep 7200
1032024-12-09T15:07:47.848Z+ psrinfo -v
1042024-12-09T15:07:47.851Z+ df -h
1052024-12-09T15:07:47.857Z+ prstat -d d -mLc 1
1062024-12-09T15:07:47.860Z+ iostat -T d -xn 1
1072024-12-09T15:07:47.860Z+ mpstat -T d 1
1082024-12-09T15:07:47.860Z+ vmstat -T d -p 1
1092024-12-09T15:07:47.863Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102024-12-09T15:07:47.863Z+ banner test_up_unencrypted
1112024-12-09T15:07:47.863Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122024-12-09T15:07:47.863Z
1132024-12-09T15:07:47.863Z ##### ###### #### ##### # # ##### # # # #
1142024-12-09T15:07:47.863Z # # # # # # # # # # ## #
1152024-12-09T15:07:47.863Z # ##### #### # # # # # # # # # #
1162024-12-09T15:07:47.863Z # # # # # # ##### # # # # #
1172024-12-09T15:07:47.863Z # # # # # # # # # # # ##
1182024-12-09T15:07:47.863Z # ###### #### # ####### #### # ####### #### # #
1192024-12-09T15:07:47.863Z
1202024-12-09T15:07:47.863Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N unencrypted
1212024-12-09T15:07:47.874Z/input/build/work
1222024-12-09T15:07:47.886ZTurn off color for downstairs dump
1232024-12-09T15:07:47.888Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1242024-12-09T15:07:47.888ZCreating 3 downstairs regions
1252024-12-09T15:07:48.213ZStarting 3 downstairs
1262024-12-09T15:07:48.217Zdsc started at PID: 1186
1272024-12-09T15:07:53.396ZDisable automatic restart on all downstairs
1282024-12-09T15:07:53.450Z
1292024-12-09T15:07:53.450ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1302024-12-09T15:07:53.454ZRunning test: span
1312024-12-09T15:07:57.632ZCompleted test: span
1322024-12-09T15:08:02.727ZRunning test: big
1332024-12-09T15:08:06.818ZCompleted test: big
1342024-12-09T15:08:11.825ZRunning test: dep
1352024-12-09T15:08:24.473ZCompleted test: dep
1362024-12-09T15:08:29.479ZRunning test: balloon
1372024-12-09T15:08:39.151ZCompleted test: balloon
1382024-12-09T15:08:44.158ZRunning test: deactivate
1392024-12-09T15:09:38.403ZCompleted test: deactivate
1402024-12-09T15:09:43.411ZRunning hammer
1412024-12-09T15:09:46.564ZRun repair tests
1422024-12-09T15:09:46.568Z/var/tmp/bins/crutest fill -g 31 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1432024-12-09T15:09:46.627Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.450526189Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"task":"crutest"}
1442024-12-09T15:09:46.629Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.451060407Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"task":"crutest"}
1452024-12-09T15:09:46.629Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.452713062Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"task":"crutest"}
1462024-12-09T15:09:46.629Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.452908455Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223}
1472024-12-09T15:09:46.629Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.45294755Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223}
1482024-12-09T15:09:46.629Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.452963417Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223}
1492024-12-09T15:09:46.629Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.452979715Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223}
1502024-12-09T15:09:46.629Z{"msg":"Crucible c7c4ebba-1730-4756-a13d-fd126858ac4d has session id: 436c6e86-2756-4a22-ab58-ef3137e4a18b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.453381031Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1512024-12-09T15:09:46.629Z{"msg":"Upstairs opts: Upstairs UUID: c7c4ebba-1730-4756-a13d-fd126858ac4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.453483866Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1522024-12-09T15:09:46.629Z{"msg":"Crucible stats registered with UUID: c7c4ebba-1730-4756-a13d-fd126858ac4d","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.453523271Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1532024-12-09T15:09:46.662Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486187443Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"task":"crutest"}
1542024-12-09T15:09:46.662Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486225077Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223}
1552024-12-09T15:09:46.666Z{"msg":"c7c4ebba-1730-4756-a13d-fd126858ac4d active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486308156Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1562024-12-09T15:09:46.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486344741Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"0","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1572024-12-09T15:09:46.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486373047Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"0","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1582024-12-09T15:09:46.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486404103Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"1","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1592024-12-09T15:09:46.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486443357Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"1","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1602024-12-09T15:09:46.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486580499Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"2","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1612024-12-09T15:09:46.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486628512Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"2","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1622024-12-09T15:09:46.666Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486672276Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1632024-12-09T15:09:46.666Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486699053Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1642024-12-09T15:09:46.666Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486740387Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1652024-12-09T15:09:46.666Z{"msg":"ds_connection connected from Ok(127.0.0.1:49702)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486774483Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1662024-12-09T15:09:46.666Z{"msg":"ds_connection connected from Ok(127.0.0.1:35199)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486799749Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1672024-12-09T15:09:46.666Z{"msg":"ds_connection connected from Ok(127.0.0.1:38494)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.486837114Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1682024-12-09T15:09:46.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.487371971Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"2","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1692024-12-09T15:09:46.666Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.487410636Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"2","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1702024-12-09T15:09:46.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.487465418Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"0","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1712024-12-09T15:09:46.667Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.487561015Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"0","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1722024-12-09T15:09:46.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.48760184Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"1","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1732024-12-09T15:09:46.667Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.487633215Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"1","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1742024-12-09T15:09:46.667Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488060677Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"2","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1752024-12-09T15:09:46.667Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488104671Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"1","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1762024-12-09T15:09:46.667Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488129228Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"0","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1772024-12-09T15:09:46.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.48847954Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"1","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1782024-12-09T15:09:46.667Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488524454Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"1","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1792024-12-09T15:09:46.667Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488541412Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1802024-12-09T15:09:46.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.48855475Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"0","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1812024-12-09T15:09:46.667Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488568898Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"0","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1822024-12-09T15:09:46.667Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488582536Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1832024-12-09T15:09:46.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488595124Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"2","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1842024-12-09T15:09:46.667Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488617141Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"2","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1852024-12-09T15:09:46.667Z{"msg":"[0]R flush_numbers: [1500, 1500, 1500, 1500, 1500]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.48863159Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1862024-12-09T15:09:46.667Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488652707Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1872024-12-09T15:09:46.667Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488666875Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1882024-12-09T15:09:46.667Z{"msg":"[1]R flush_numbers: [1500, 1500, 1500, 1500, 1500]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488687322Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1892024-12-09T15:09:46.667Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.48870145Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1902024-12-09T15:09:46.667Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488721507Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1912024-12-09T15:09:46.667Z{"msg":"[2]R flush_numbers: [1500, 1500, 1500, 1500, 1500]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488735585Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1922024-12-09T15:09:46.667Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488756102Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1932024-12-09T15:09:46.667Z{"msg":"[2]R dirty: [false, false, false, false, false]"Wait for a query_work_queue command to finish before sending IO
1942024-12-09T15:09:46.667Z,"v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488837882Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1952024-12-09T15:09:46.667Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488872937Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1962024-12-09T15:09:46.667Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488888384Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1972024-12-09T15:09:46.667Z{"msg":"Next flush: 1501","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488915911Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1982024-12-09T15:09:46.667Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488931539Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"":"downstairs","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
1992024-12-09T15:09:46.667Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488951946Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
2002024-12-09T15:09:46.667Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488964514Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"0","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
2012024-12-09T15:09:46.667Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488984771Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"1","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
2022024-12-09T15:09:46.667Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.488998409Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"client":"2","":"downstairs","session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
2032024-12-09T15:09:46.667Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.489018997Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
2042024-12-09T15:09:46.667Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.489031865Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
2052024-12-09T15:09:46.667Z{"msg":"c7c4ebba-1730-4756-a13d-fd126858ac4d is now active with session: 436c6e86-2756-4a22-ab58-ef3137e4a18b","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.489051762Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
2062024-12-09T15:09:46.667Z{"msg":"Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2072024-12-09T15:09:46.668ZSet Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.4890686Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223,"session_id":"436c6e86-2756-4a22-ab58-ef3137e4a18b"}
2082024-12-09T15:09:46.668Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:45.489088317Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223}
2092024-12-09T15:09:46.668ZFill test
2102024-12-09T15:09:46.671ZRead and Verify all blocks (0..50 range:false)
2112024-12-09T15:09:46.676ZWrote out file "/var/tmp/test_up-build/verify_file"
2122024-12-09T15:09:46.676ZCLIENT: Tests done. All submitted work has been ACK'd
2132024-12-09T15:09:46.680Z----------------------------------------------------------------
2142024-12-09T15:09:46.680Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2152024-12-09T15:09:46.680ZStates: Active Active Active
2162024-12-09T15:09:46.680ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2172024-12-09T15:09:46.680Z 1002 Acked Read 50 Done Done Done false
2182024-12-09T15:09:46.680Z STATES DS:0 DS:1 DS:2 TOTAL
2192024-12-09T15:09:46.680Z Sent 0 0 0 0
2202024-12-09T15:09:46.680Z Done 1 1 1 3
2212024-12-09T15:09:46.680Z Skipped 0 0 0 0
2222024-12-09T15:09:46.680Z Error 0 0 0 0
2232024-12-09T15:09:46.680ZLast Flush: 1001 1001 1001
2242024-12-09T15:09:46.680ZDownstairs last five completed: 1001 1000
2252024-12-09T15:09:46.680ZUpstairs last five completed: 1002 1001 1000
2262024-12-09T15:09:46.680ZCLIENT: Up:0 ds:1 act:3
2272024-12-09T15:09:50.677Z----------------------------------------------------------------
2282024-12-09T15:09:50.678Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2292024-12-09T15:09:50.678ZDownstairs last five completed: 1003 1002 1001 1000
2302024-12-09T15:09:50.678ZUpstairs last five completed: 1003 1002 1001 1000
2312024-12-09T15:09:50.680ZCLIENT: Up:0 ds:0 act:3
2322024-12-09T15:09:50.680ZCLIENT: All crucible jobs finished, exiting program
2332024-12-09T15:09:50.680Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:49.501544247Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1223}
2342024-12-09T15:09:50.689ZRepair setup passed
2352024-12-09T15:09:50.693ZCopy the region for /var/tmp/test_up-build/8830
2362024-12-09T15:09:50.696Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2372024-12-09T15:09:50.698Z/var/tmp/bins/crutest fill -g 32 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2382024-12-09T15:09:50.753Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.576484471Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"task":"crutest"}
2392024-12-09T15:09:50.755Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.577037905Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"task":"crutest"}
2402024-12-09T15:09:50.755Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.57869546Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"task":"crutest"}
2412024-12-09T15:09:50.755Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.578902011Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229}
2422024-12-09T15:09:50.755Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.578941386Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229}
2432024-12-09T15:09:50.755Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.578976891Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229}
2442024-12-09T15:09:50.755Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.579014006Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229}
2452024-12-09T15:09:50.758Z{"msg":"Crucible fee2e386-185d-465a-9d21-db53b0928a86 has session id: 5a14aba4-a7cc-420c-8b5c-97b782bed4b7","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.579407962Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2462024-12-09T15:09:50.758Z{"msg":"Upstairs opts: Upstairs UUID: fee2e386-185d-465a-9d21-db53b0928a86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.579454066Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2472024-12-09T15:09:50.758Z{"msg":"Crucible stats registered with UUID: fee2e386-185d-465a-9d21-db53b0928a86","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.579493681Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2482024-12-09T15:09:50.788Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612452462Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"task":"crutest"}
2492024-12-09T15:09:50.789Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612498106Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229}
2502024-12-09T15:09:50.791Z{"msg":"fee2e386-185d-465a-9d21-db53b0928a86 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612532171Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2512024-12-09T15:09:50.791Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.61261467Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"0","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2522024-12-09T15:09:50.792Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.61268838Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"0","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2532024-12-09T15:09:50.792Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612715826Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"1","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2542024-12-09T15:09:50.792Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612751431Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"1","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2552024-12-09T15:09:50.792Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612780827Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"2","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2562024-12-09T15:09:50.792Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612816212Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"2","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2572024-12-09T15:09:50.792Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612840029Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2582024-12-09T15:09:50.792Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612882014Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2592024-12-09T15:09:50.792Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.61290907Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2602024-12-09T15:09:50.792Z{"msg":"ds_connection connected from Ok(127.0.0.1:42433)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612942215Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2612024-12-09T15:09:50.792Z{"msg":"ds_connection connected from Ok(127.0.0.1:44591)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612965592Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2622024-12-09T15:09:50.792Z{"msg":"ds_connection connected from Ok(127.0.0.1:55755)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.612991259Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2632024-12-09T15:09:50.792Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.613634291Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"1","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2642024-12-09T15:09:50.792Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.613673606Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"1","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2652024-12-09T15:09:50.793Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.613701962Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"0","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2662024-12-09T15:09:50.793Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.613733138Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"0","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2672024-12-09T15:09:50.793Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.613772032Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"2","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2682024-12-09T15:09:50.793Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.613812967Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"2","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2692024-12-09T15:09:50.793Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614381069Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"0","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2702024-12-09T15:09:50.793Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614424983Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"2","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2712024-12-09T15:09:50.793Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614469037Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"1","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2722024-12-09T15:09:50.793Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614796103Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"1","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2732024-12-09T15:09:50.793Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614843836Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"1","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2742024-12-09T15:09:50.793Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614869253Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2752024-12-09T15:09:50.793Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.61489206Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"2","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2762024-12-09T15:09:50.793Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614907068Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"2","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2772024-12-09T15:09:50.793Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614921816Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2782024-12-09T15:09:50.793Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614934224Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"0","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2792024-12-09T15:09:50.793Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614948352Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"0","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2802024-12-09T15:09:50.793Z{"msg":"[0]R flush_numbers: [1501, 1501, 1501, 1501, 1501]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.61496167Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2812024-12-09T15:09:50.793Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.614983157Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2822024-12-09T15:09:50.793Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615002994Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2832024-12-09T15:09:50.793Z{"msg":"[1]R flush_numbers: [1501, 1501, 1501, 1501, 1501]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615024762Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2842024-12-09T15:09:50.793Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.61503913Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2852024-12-09T15:09:50.793Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615062746Z"Wait for a query_work_queue command to finish before sending IO
2862024-12-09T15:09:50.793Z,"hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2872024-12-09T15:09:50.793Z{"msg":"[2]R flush_numbers: [1501, 1501, 1501, 1501, 1501]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615149315Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2882024-12-09T15:09:50.797Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.61518089Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2892024-12-09T15:09:50.797Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615206537Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2902024-12-09T15:09:50.797Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615247761Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2912024-12-09T15:09:50.797Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615290835Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2922024-12-09T15:09:50.797Z{"msg":"Next flush: 1502","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615322091Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2932024-12-09T15:09:50.797Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615362416Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"":"downstairs","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2942024-12-09T15:09:50.797Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.61540146Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":Disk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2952024-12-09T15:09:50.797Z1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2962024-12-09T15:09:50.797Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615440495Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"0","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2972024-12-09T15:09:50.797Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615465702Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"1","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2982024-12-09T15:09:50.797Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.61547964Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"client":"2","":"downstairs","session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
2992024-12-09T15:09:50.797Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615501137Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
3002024-12-09T15:09:50.797Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615514035Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
3012024-12-09T15:09:50.797Z{"msg":"fee2e386-185d-465a-9d21-db53b0928a86 is now active with session: 5a14aba4-a7cc-420c-8b5c-97b782bed4b7","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615533902Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
3022024-12-09T15:09:50.797Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615546571Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229,"session_id":"5a14aba4-a7cc-420c-8b5c-97b782bed4b7"}
3032024-12-09T15:09:50.797Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:49.615559229Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229}
3042024-12-09T15:09:50.797ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
3052024-12-09T15:09:50.797ZFill test
3062024-12-09T15:09:50.800ZRead and Verify all blocks (0..50 range:false)
3072024-12-09T15:09:50.803ZWrote out file "/var/tmp/test_up-build/verify_file"
3082024-12-09T15:09:50.803ZCLIENT: Tests done. All submitted work has been ACK'd
3092024-12-09T15:09:50.805Z----------------------------------------------------------------
3102024-12-09T15:09:50.806Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3112024-12-09T15:09:50.806ZStates: Active Active Active
3122024-12-09T15:09:50.806ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3132024-12-09T15:09:50.806Z 1002 Acked Read 50 Done Done Done false
3142024-12-09T15:09:50.806Z STATES DS:0 DS:1 DS:2 TOTAL
3152024-12-09T15:09:50.806Z Sent 0 0 0 0
3162024-12-09T15:09:50.806Z Done 1 1 1 3
3172024-12-09T15:09:50.806Z Skipped 0 0 0 0
3182024-12-09T15:09:50.806Z Error 0 0 0 0
3192024-12-09T15:09:50.806ZLast Flush: 1001 1001 1001
3202024-12-09T15:09:50.806ZDownstairs last five completed: 1001 1000
3212024-12-09T15:09:50.806ZUpstairs last five completed: 1002 1001 1000
3222024-12-09T15:09:50.806ZCLIENT: Up:0 ds:1 act:3
3232024-12-09T15:09:54.805Z----------------------------------------------------------------
3242024-12-09T15:09:54.805Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3252024-12-09T15:09:54.805ZDownstairs last five completed: 1003 1002 1001 1000
3262024-12-09T15:09:54.805ZUpstairs last five completed: 1003 1002 1001 1000
3272024-12-09T15:09:54.805ZCLIENT: Up:0 ds:0 act:3
3282024-12-09T15:09:54.805ZCLIENT: All crucible jobs finished, exiting program
3292024-12-09T15:09:54.808Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:53.627886106Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1229}
3302024-12-09T15:09:54.812ZRepair part 1 passed
3312024-12-09T15:09:54.815Z
3322024-12-09T15:09:54.815ZKill the current downstairs
3332024-12-09T15:09:54.981ZDownstairs 2 stopped
3342024-12-09T15:09:54.981Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3352024-12-09T15:09:54.984ZNow put back the original so we have a mismatch
3362024-12-09T15:09:54.984Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3372024-12-09T15:09:54.987ZRestart downstairs with old directory
3382024-12-09T15:09:55.038Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3392024-12-09T15:09:55.058ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3402024-12-09T15:09:55.061Z 0 000-009 32 32 31 1502 1502 1501 F F F <---
3412024-12-09T15:09:55.061Z 1 010-019 32 32 31 1502 1502 1501 F F F <---
3422024-12-09T15:09:55.061Z 2 020-029 32 32 31 1502 1502 1501 F F F <---
3432024-12-09T15:09:55.061Z 3 030-039 32 32 31 1502 1502 1501 F F F <---
3442024-12-09T15:09:55.061Z 4 040-049 32 32 31 1502 1502 1501 F F F <---
3452024-12-09T15:09:55.061ZMax gen: 32, Max flush: 1502
3462024-12-09T15:09:55.061ZError: Difference in extent metadata found!
3472024-12-09T15:09:55.062Zdump test found error as expected
3482024-12-09T15:09:55.062Z
3492024-12-09T15:09:55.062Z
3502024-12-09T15:09:55.065Z/var/tmp/bins/crutest verify --range -g 33 -q --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3512024-12-09T15:09:55.118Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.940754173Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"task":"crutest"}
3522024-12-09T15:09:55.121Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.941294909Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"task":"crutest"}
3532024-12-09T15:09:55.121Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.942958993Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"task":"crutest"}
3542024-12-09T15:09:55.121Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.943153846Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237}
3552024-12-09T15:09:55.121Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2978a42101da810efa0e51ea3f370f1f90c0069b\",\n git_commit_timestamp: \"2024-12-09T14:48:58.000000000Z\",\n git_branch: \"mkeeter/simplify-faults\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.943194711Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237}
3562024-12-09T15:09:55.121Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.943275489Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237}
3572024-12-09T15:09:55.121Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.943314844Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237}
3582024-12-09T15:09:55.121Z{"msg":"Crucible 693b874b-0c6c-4f0b-8524-ba1cf6a651b4 has session id: 1aa6a3fa-bf48-45eb-9da2-46850455e887","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.943670756Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3592024-12-09T15:09:55.124Z{"msg":"Upstairs opts: Upstairs UUID: 693b874b-0c6c-4f0b-8524-ba1cf6a651b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.943737217Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3602024-12-09T15:09:55.124Z{"msg":"Crucible stats registered with UUID: 693b874b-0c6c-4f0b-8524-ba1cf6a651b4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.943824695Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3612024-12-09T15:09:55.154Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.976634016Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"task":"crutest"}
3622024-12-09T15:09:55.154Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.97668206Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237}
3632024-12-09T15:09:55.154Z{"msg":"693b874b-0c6c-4f0b-8524-ba1cf6a651b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.976707336Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3642024-12-09T15:09:55.156Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.976741311Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3652024-12-09T15:09:55.156Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.976792655Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3662024-12-09T15:09:55.156Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.97682875Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3672024-12-09T15:09:55.156Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.976871044Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3682024-12-09T15:09:55.156Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.97690027Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3692024-12-09T15:09:55.156Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.976936685Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3702024-12-09T15:09:55.156Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.976963691Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3712024-12-09T15:09:55.156Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.976998467Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3722024-12-09T15:09:55.156Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.977028823Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3732024-12-09T15:09:55.156Z{"msg":"ds_connection connected from Ok(127.0.0.1:39646)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.977056329Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3742024-12-09T15:09:55.158Z{"msg":"ds_connection connected from Ok(127.0.0.1:56209)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.977100313Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3752024-12-09T15:09:55.158Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:53.977132298Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3762024-12-09T15:09:55.158Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:53.977172423Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3772024-12-09T15:09:55.158Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:09:53.977261671Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3782024-12-09T15:09:55.158Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.977313054Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3792024-12-09T15:09:55.158Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.977816195Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3802024-12-09T15:09:55.158Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.977869338Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3812024-12-09T15:09:55.158Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.977916002Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3822024-12-09T15:09:55.158Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.977947127Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3832024-12-09T15:09:55.158Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.978522799Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3842024-12-09T15:09:55.158Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.978566003Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3852024-12-09T15:09:55.158Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.978929844Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3862024-12-09T15:09:55.158Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.978966139Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3872024-12-09T15:09:55.158Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.978982576Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3882024-12-09T15:09:55.158Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.978997784Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3892024-12-09T15:09:55.158Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.979033399Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3902024-12-09T15:09:55.158Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:09:53.979050647Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3912024-12-09T15:10:05.158Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.979149488Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3922024-12-09T15:10:05.164Z{"msg":"ds_connection connected from Ok(127.0.0.1:57210)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.979362859Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3932024-12-09T15:10:05.164Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.980204164Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3942024-12-09T15:10:05.164Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.980246088Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3952024-12-09T15:10:05.164Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.980716784Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3962024-12-09T15:10:05.164Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981085854Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3972024-12-09T15:10:05.164Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981121169Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3982024-12-09T15:10:05.164Z{"msg":"[0]R flush_numbers: [1502, 1502, 1502, 1502, 1502]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981137477Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
3992024-12-09T15:10:05.164Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981160734Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4002024-12-09T15:10:05.164Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981176092Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4012024-12-09T15:10:05.164Z{"msg":"[1]R flush_numbers: [1502, 1502, 1502, 1502, 1502]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981197929Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4022024-12-09T15:10:05.164Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981212547Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4032024-12-09T15:10:05.164Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981233124Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4042024-12-09T15:10:05.164Z{"msg":"[2]R flush_numbers: [1501, 1501, 1501, 1501, 1501]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981247832Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4052024-12-09T15:10:05.164Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.98126115Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4062024-12-09T15:10:05.164Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981274818Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4072024-12-09T15:10:05.164Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981295615Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4082024-12-09T15:10:05.164Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981309553Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4092024-12-09T15:10:05.164Z{"msg":"Next flush: 1503","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.9813307Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4102024-12-09T15:10:05.164Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981345218Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4112024-12-09T15:10:05.164Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981366356Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4122024-12-09T15:10:05.164Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981381864Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4132024-12-09T15:10:05.164Z{"msg":"extent:0 flush: 1502 1502 1501 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.98140329Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4142024-12-09T15:10:05.165Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981419268Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4152024-12-09T15:10:05.165Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981440785Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4162024-12-09T15:10:05.165Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981456073Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4172024-12-09T15:10:05.165Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.98147718Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4182024-12-09T15:10:05.165Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981491718Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4192024-12-09T15:10:05.165Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981523194Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4202024-12-09T15:10:05.165Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981546131Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4212024-12-09T15:10:05.165Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981560769Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4222024-12-09T15:10:05.165Z{"msg":"extent:1 flush: 1502 1502 1501 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981582246Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4232024-12-09T15:10:05.165Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981598304Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4242024-12-09T15:10:05.165Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981620001Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4252024-12-09T15:10:05.165Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981635309Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4262024-12-09T15:10:05.165Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981656386Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4272024-12-09T15:10:05.165Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981671014Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4282024-12-09T15:10:05.165Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981693071Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4292024-12-09T15:10:05.165Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981707269Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4302024-12-09T15:10:05.165Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981728246Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4312024-12-09T15:10:05.165Z{"msg":"extent:2 flush: 1502 1502 1501 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981743854Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4322024-12-09T15:10:05.165Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981764881Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4332024-12-09T15:10:05.165Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981779619Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4342024-12-09T15:10:05.165Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981801336Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4352024-12-09T15:10:05.165Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981816474Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4362024-12-09T15:10:05.165Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981837591Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4372024-12-09T15:10:05.165Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981853239Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4382024-12-09T15:10:05.165Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981882275Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4392024-12-09T15:10:05.165Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.98191866Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4402024-12-09T15:10:05.165Z{"msg":"extent:3 flush: 1502 1502 1501 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981955625Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4412024-12-09T15:10:05.165Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.981983271Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4422024-12-09T15:10:05.165Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982010268Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4432024-12-09T15:10:05.165Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982039314Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4442024-12-09T15:10:05.165Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982075909Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4452024-12-09T15:10:05.165Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982093087Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4462024-12-09T15:10:05.165Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982109325Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4472024-12-09T15:10:05.165Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982131051Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4482024-12-09T15:10:05.165Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982146759Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4492024-12-09T15:10:05.165Z{"msg":"extent:4 flush: 1502 1502 1501 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982161527Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4502024-12-09T15:10:05.165Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982175945Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4512024-12-09T15:10:05.165Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982198442Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4522024-12-09T15:10:05.165Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.98221367Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4532024-12-09T15:10:05.166Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982235037Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4542024-12-09T15:10:05.166Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982250415Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4552024-12-09T15:10:05.166Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982266573Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4562024-12-09T15:10:05.166Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982280111Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4572024-12-09T15:10:05.166Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982300848Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4582024-12-09T15:10:05.166Z{"msg":"Full repair list: {ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982318596Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4592024-12-09T15:10:05.166Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1503, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1503, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1503, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1503, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1503, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982392036Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4602024-12-09T15:10:05.166Z{"msg":"starting reconciliation 1f122e7f-d74f-4522-8425-b1a29ce567c8: found 20 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982408564Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4612024-12-09T15:10:05.166Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.982422272Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4622024-12-09T15:10:05.166Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.98243751Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4632024-12-09T15:10:05.166Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.982519918Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4642024-12-09T15:10:05.166Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.982867211Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4652024-12-09T15:10:05.169Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.982922034Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4662024-12-09T15:10:05.169Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.99014588Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4672024-12-09T15:10:05.172Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.990553214Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4682024-12-09T15:10:05.172Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.990818668Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4692024-12-09T15:10:05.172Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.991151203Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4702024-12-09T15:10:05.172Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.991189408Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4712024-12-09T15:10:05.175Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.994160603Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4722024-12-09T15:10:05.175Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.994580586Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4732024-12-09T15:10:05.175Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.994822603Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4742024-12-09T15:10:05.175Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.995148258Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4752024-12-09T15:10:05.175Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.995230227Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4762024-12-09T15:10:05.178Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.997810926Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4772024-12-09T15:10:05.178Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.998202242Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4782024-12-09T15:10:05.178Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.998432911Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4792024-12-09T15:10:05.178Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:03.998705804Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4802024-12-09T15:10:05.178Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:03.998749488Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4812024-12-09T15:10:05.181Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:04.001471007Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4822024-12-09T15:10:05.181Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:04.001869333Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4832024-12-09T15:10:05.181Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:04.002119599Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4842024-12-09T15:10:05.183Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:04.002484589Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4852024-12-09T15:10:05.183Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:04.002532963Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4862024-12-09T15:10:05.186Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:04.005462124Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4872024-12-09T15:10:05.186Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:04.005823455Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4882024-12-09T15:10:05.186Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:04.005863659Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"0","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4892024-12-09T15:10:05.186Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:04.005880857Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"1","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4902024-12-09T15:10:05.186Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:04.005904044Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"client":"2","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4912024-12-09T15:10:05.186Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:04.00592681Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4922024-12-09T15:10:05.186Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:04.005941039Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4932024-12-09T15:10:05.187ZWait for a query_work_queue command to finish before sending IO
4942024-12-09T15:10:05.187Z{"msg":"693b874b-0c6c-4f0b-8524-ba1cf6a651b4 is now active with session: 1aa6a3fa-bf48-45eb-9da2-46850455e887","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:04.005970615Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4952024-12-09T15:10:05.187Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-12-09T15:10:04.005992102Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237}
4962024-12-09T15:10:05.187Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-12-09T15:10:04.00600551Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237,"reconcile":"1f122e7f-d74f-4522-8425-b1a29ce567c8","":"downstairs","":"downstairs","session_id":"1aa6a3fa-bf48-45eb-9da2-46850455e887"}
4972024-12-09T15:10:05.187ZDisk: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4982024-12-09T15:10:05.187ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4992024-12-09T15:10:05.187ZRead and Verify all blocks (0..50 range:true)
5002024-12-09T15:10:05.191ZWrote out file "/var/tmp/test_up-build/verify_file"
5012024-12-09T15:10:05.191ZVerify test completed
5022024-12-09T15:10:05.194ZWrote out file "/var/tmp/test_up-build/verify_file"
5032024-12-09T15:10:05.194ZCLIENT: Tests done. All submitted work has been ACK'd
5042024-12-09T15:10:05.194Z----------------------------------------------------------------
5052024-12-09T15:10:05.194Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5062024-12-09T15:10:05.194ZStates: Active Active Active
5072024-12-09T15:10:05.194ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5082024-12-09T15:10:05.194Z 1000 Acked Read 50 Done Done Done false
5092024-12-09T15:10:05.194Z STATES DS:0 DS:1 DS:2 TOTAL
5102024-12-09T15:10:05.194Z Sent 0 0 0 0
5112024-12-09T15:10:05.194Z Done 1 1 1 3
5122024-12-09T15:10:05.194Z Skipped 0 0 0 0
5132024-12-09T15:10:05.194Z Error 0 0 0 0
5142024-12-09T15:10:05.194ZLast Flush: 0 0 0
5152024-12-09T15:10:05.194ZDownstairs last five completed:
5162024-12-09T15:10:05.194ZUpstairs last five completed: 1000
5172024-12-09T15:10:05.194ZCLIENT: Up:0 ds:1 act:3
5182024-12-09T15:10:09.193Z----------------------------------------------------------------
5192024-12-09T15:10:09.193Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5202024-12-09T15:10:09.193ZDownstairs last five completed: 1001 1000
5212024-12-09T15:10:09.193ZUpstairs last five completed: 1001 1000
5222024-12-09T15:10:09.194ZCLIENT: Up:0 ds:0 act:3
5232024-12-09T15:10:09.194ZCLIENT: All crucible jobs finished, exiting program
5242024-12-09T15:10:09.196Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-12-09T15:10:08.014114431Z","hostname":"w-01JENZA9QXPWE9QFN5XVGTSRPW","pid":1237}
5252024-12-09T15:10:09.201ZRepair part 2 passed
5262024-12-09T15:10:09.204Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5272024-12-09T15:10:09.218ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5282024-12-09T15:10:09.218Z 0 000-009 32 32 32 1502 1502 1502 F F F
5292024-12-09T15:10:09.222Z 1 010-019 32 32 32 1502 1502 1502 F F F
5302024-12-09T15:10:09.222Z 2 020-029 32 32 32 1502 1502 1502 F F F
5312024-12-09T15:10:09.222Z 3 030-039 32 32 32 1502 1502 1502 F F F
5322024-12-09T15:10:09.222Z 4 040-049 32 32 32 1502 1502 1502 F F F
5332024-12-09T15:10:09.222ZMax gen: 32, Max flush: 1502
5342024-12-09T15:10:09.222Zdump test passed
5352024-12-09T15:10:09.222Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5362024-12-09T15:10:09.236Z Extent 1
5372024-12-09T15:10:09.236ZGEN 32 32 32
5382024-12-09T15:10:09.236ZFLUSH_ID 1502 1502 1502
5392024-12-09T15:10:09.236ZDIRTY
5402024-12-09T15:10:09.236Z
5412024-12-09T15:10:09.239ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5422024-12-09T15:10:09.239Z010 A A A A A A
5432024-12-09T15:10:09.239Z011 A A A A A A
5442024-12-09T15:10:09.242Z012 A A A A A A
5452024-12-09T15:10:09.242Z013 A A A A A A
5462024-12-09T15:10:09.245Z014 A A A A A A
5472024-12-09T15:10:09.245Z015 A A A A A A
5482024-12-09T15:10:09.245Z016 A A A A A A
5492024-12-09T15:10:09.248Z017 A A A A A A
5502024-12-09T15:10:09.248Z018 A A A A A A
5512024-12-09T15:10:09.253Z019 A A A A A A
5522024-12-09T15:10:09.253Zdump extent test passed
5532024-12-09T15:10:09.253Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5542024-12-09T15:10:09.266ZExtent 2 Block in extent 0 Actual block 20
5552024-12-09T15:10:09.266Z
5562024-12-09T15:10:09.268Z DATA SHA256 VER
5572024-12-09T15:10:09.268Z------ ---------------------------------------------------------------- ---
5582024-12-09T15:10:09.268Z 0 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5592024-12-09T15:10:09.268Z 1 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5602024-12-09T15:10:09.269Z 2 ad3d0e88276af7eb1f904413516e1a8c1f9c2fa8b17b86b2d842d08303e90aa2 A
5612024-12-09T15:10:09.269Z
5622024-12-09T15:10:09.269ZNONCES 0 1 2 DIFF
5632024-12-09T15:10:09.269Z------ ------------------------ ------------------------ ------------------------ -----
5642024-12-09T15:10:09.269Z
5652024-12-09T15:10:09.269Z TAGS 0 1 2 DIFF
5662024-12-09T15:10:09.269Z------ -------------------------------- -------------------------------- -------------------------------- -----
5672024-12-09T15:10:09.269Z
5682024-12-09T15:10:09.269ZHASHES 0 1 2 DIFF
5692024-12-09T15:10:09.269Z------ ---------------- ---------------- ---------------- -----
5702024-12-09T15:10:09.269Z 0 42916ab6fa364c70 42916ab6fa364c70 42916ab6fa364c70
5712024-12-09T15:10:09.269Z
5722024-12-09T15:10:09.272Zdump block test passed
5732024-12-09T15:10:09.272ZInitial upstairs tests have completed, stopping all downstairs
5742024-12-09T15:10:14.335ZCreating 4 larger downstairs regions
5752024-12-09T15:10:14.440ZStarting 4 downstairs
5762024-12-09T15:10:19.448Zdsc restarted at PID: 1250
5772024-12-09T15:10:19.678ZNow do the replace-reconcile test
5782024-12-09T15:10:19.736ZUsing 8840 for the replacement port
5792024-12-09T15:12:53.187ZCompleted test: replace-reconcile
5802024-12-09T15:12:53.189ZNow do the replace-before-active test
5812024-12-09T15:15:37.845ZCompleted test: replace-before-active
5822024-12-09T15:15:37.847ZAll tests have completed, stopping all downstairs
5832024-12-09T15:15:37.902Z
5842024-12-09T15:15:37.902ZAll Tests have passed
5852024-12-09T15:15:37.905Z7:50 Test duration
5862024-12-09T15:15:37.905Z
5872024-12-09T15:15:37.905Zreal 7:49.960290040
5882024-12-09T15:15:37.905Zuser 1:03.647347179
5892024-12-09T15:15:37.905Zsys 10.057682600
5902024-12-09T15:15:37.905Ztrap 0.086342931
5912024-12-09T15:15:37.905Ztflt 0.024190186
5922024-12-09T15:15:37.905Zdflt 0.009658620
5932024-12-09T15:15:37.905Zkflt 0.001203273
5942024-12-09T15:15:37.905Zlock 5:43:06.794175401
5952024-12-09T15:15:37.905Zslp 24:32.708549395
5962024-12-09T15:15:37.905Zlat 7.725258149
5972024-12-09T15:15:37.905Zstop 0.124878125
5982024-12-09T15:15:37.905Z+ echo 'test-up-unencrypted ends'
5992024-12-09T15:15:37.905Ztest-up-unencrypted ends
6002024-12-09T15:15:42.908Zprocess exited: duration 478095 ms, exit code 0
6012024-12-09T15:15:42.908Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6022024-12-09T15:16:42.945Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6032024-12-09T15:16:42.945Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6042024-12-09T15:16:42.956Zfound 14 output files
6052024-12-09T15:16:42.956Zuploading: /tmp/test_up-build/dsc-out.txt (37039 bytes)
6062024-12-09T15:16:43.970Zuploaded: /tmp/test_up-build/dsc-out.txt
6072024-12-09T15:16:43.970Zuploading: /tmp/test_up-build/test_up_out.txt (11072832 bytes)
6082024-12-09T15:16:45.219Zuploaded: /tmp/test_up-build/test_up_out.txt
6092024-12-09T15:16:45.222Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (253711 bytes)
6102024-12-09T15:16:46.243Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6112024-12-09T15:16:46.245Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144657 bytes)
6122024-12-09T15:16:47.264Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6132024-12-09T15:16:47.269Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251108 bytes)
6142024-12-09T15:16:48.288Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6152024-12-09T15:16:48.291Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139418 bytes)
6162024-12-09T15:16:49.307Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6172024-12-09T15:16:49.309Zuploading: /tmp/debug/df.txt (1270 bytes)
6182024-12-09T15:16:50.315Zuploaded: /tmp/debug/df.txt
6192024-12-09T15:16:50.318Zuploading: /tmp/debug/dtrace.txt (393773 bytes)
6202024-12-09T15:16:50.341Zupload warning: file "/tmp/debug/dtrace.txt" changed size mid upload: 393773 -> 478799
6212024-12-09T15:16:51.348Zuploaded: /tmp/debug/dtrace.txt
6222024-12-09T15:16:51.350Zuploading: /tmp/debug/iostat.txt (161347 bytes)
6232024-12-09T15:16:51.364Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 161347 -> 163755
6242024-12-09T15:16:52.371Zuploaded: /tmp/debug/iostat.txt
6252024-12-09T15:16:52.373Zuploading: /tmp/debug/mpstat.txt (391285 bytes)
6262024-12-09T15:16:52.395Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 391285 -> 397855
6272024-12-09T15:16:53.402Zuploaded: /tmp/debug/mpstat.txt
6282024-12-09T15:16:53.404Zuploading: /tmp/debug/paging.txt (70708 bytes)
6292024-12-09T15:16:53.411Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 70708 -> 72026
6302024-12-09T15:16:54.417Zuploaded: /tmp/debug/paging.txt
6312024-12-09T15:16:54.420Zuploading: /tmp/debug/prstat.txt (734887 bytes)
6322024-12-09T15:16:54.450Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 734887 -> 749136
6332024-12-09T15:16:55.457Zuploaded: /tmp/debug/prstat.txt
6342024-12-09T15:16:55.457Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6352024-12-09T15:16:56.466Zuploaded: /tmp/debug/psrinfo.txt
6362024-12-09T15:16:56.468Zuploading: /tmp/debug/upstairs-info.txt (69008 bytes)
6372024-12-09T15:16:57.479Zuploaded: /tmp/debug/upstairs-info.txt