01JCPQJT737CC7FF37VXQMF9N3: test-up-encrypted

BasicConfig {
    output_rules: [
        "%/tmp/test_up*/*.txt",
        "%/tmp/test_up*/dsc/*.txt",
        "%/tmp/debug/*",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01JCPQK4T8XJ2P4V9GET8MHMXC

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-11-15T01:55:58.360Zjob dependencies complete; ready to run (waiting for 16 m 35 s)
22024-11-15T01:57:01.533Zjob assigned to worker 01JCPRHS4XZARP6XNF1VK212KM [factory aws, i-0abb130693487e95d] (queued for 1 m 3 s)
32024-11-15T01:57:08.954Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-11-15T01:57:10.067Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-11-15T01:57:10.067Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-11-15T01:57:11.126Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-11-15T01:57:11.126Zdownloading input: /input/build/work/bins/crutest.gz
82024-11-15T01:57:12.225Zdownloaded input: /input/build/work/bins/crutest.gz
92024-11-15T01:57:12.225Zdownloading input: /input/build/work/bins/dsc.gz
102024-11-15T01:57:12.900Zdownloaded input: /input/build/work/bins/dsc.gz
112024-11-15T01:57:12.904Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-11-15T01:57:12.932Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-11-15T01:57:12.932Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-11-15T01:57:12.950Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-11-15T01:57:12.950Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-11-15T01:57:12.966Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-11-15T01:57:12.966Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-11-15T01:57:12.983Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-11-15T01:57:12.983Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-11-15T01:57:12.998Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-11-15T01:57:12.998Zdownloading input: /input/build/work/scripts/test_up.sh
222024-11-15T01:57:13.016Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-11-15T01:57:13.016Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-11-15T01:57:13.037Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-11-15T01:57:13.040Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-11-15T01:57:13.671Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-11-15T01:57:13.671Zstarting task 0: "setup"
282024-11-15T01:57:13.683Z++ uname -s
292024-11-15T01:57:13.686Z+ kern=SunOS
302024-11-15T01:57:13.686Z+ case "$kern" in
312024-11-15T01:57:13.686Z+ groupadd -g 12345 build
322024-11-15T01:57:13.689Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-11-15T01:57:15.695Z+ zfs create -o mountpoint=/work rpool/work
342024-11-15T01:57:15.778Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-11-15T01:57:15.781Z+ home_fs=zfs
362024-11-15T01:57:15.781Z+ [[ zfs == autofs ]]
372024-11-15T01:57:15.781Z+ mkdir -p /home/build
382024-11-15T01:57:15.784Z+ chown build:build /home/build /work
392024-11-15T01:57:17.783Z+ chmod 0700 /home/build /work
402024-11-15T01:57:17.787Zprocess exited: duration 4116 ms, exit code 0
 
412024-11-15T01:57:17.794Zstarting task 1: "authentication"
422024-11-15T01:57:17.813Zprocess exited: duration 18 ms, exit code 0
 
432024-11-15T01:57:17.822Zstarting task 2: "build"
442024-11-15T01:57:17.825Z+ banner cores
452024-11-15T01:57:17.829Z
462024-11-15T01:57:17.829Z #### #### ##### ###### ####
472024-11-15T01:57:17.829Z # # # # # # # #
482024-11-15T01:57:17.829Z # # # # # ##### ####
492024-11-15T01:57:17.829Z # # # ##### # #
502024-11-15T01:57:17.829Z # # # # # # # # #
512024-11-15T01:57:17.829Z #### #### # # ###### ####
522024-11-15T01:57:17.829Z
532024-11-15T01:57:17.829Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-11-15T01:57:17.836Z+ echo 'input bins dir contains:'
552024-11-15T01:57:17.836Zinput bins dir contains:
562024-11-15T01:57:17.836Z+ ls -ltr /input/build/work/bins
572024-11-15T01:57:17.839Ztotal 762772
582024-11-15T01:57:17.839Z-rw-r--r-- 1 root root 109276874 Nov 15 01:57 crucible-downstairs.gz
592024-11-15T01:57:17.839Z-rw-r--r-- 1 root root 105057390 Nov 15 01:57 crucible-hammer.gz
602024-11-15T01:57:17.839Z-rw-r--r-- 1 root root 109474605 Nov 15 01:57 crutest.gz
612024-11-15T01:57:17.839Z-rw-r--r-- 1 root root 66480466 Nov 15 01:57 dsc.gz
622024-11-15T01:57:17.839Z+ banner unpack
632024-11-15T01:57:17.843Z
642024-11-15T01:57:17.843Z # # # # ##### ## #### # #
652024-11-15T01:57:17.843Z # # ## # # # # # # # # #
662024-11-15T01:57:17.843Z # # # # # # # # # # ####
672024-11-15T01:57:17.843Z # # # # # ##### ###### # # #
682024-11-15T01:57:17.843Z # # # ## # # # # # # #
692024-11-15T01:57:17.843Z #### # # # # # #### # #
702024-11-15T01:57:17.843Z
712024-11-15T01:57:17.843Z+ mkdir -p /var/tmp/bins
722024-11-15T01:57:17.843Z+ for t in "$input/bins/"*.gz
732024-11-15T01:57:17.846Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-11-15T01:57:17.846Z+ b=crucible-downstairs.gz
752024-11-15T01:57:17.846Z+ b=crucible-downstairs
762024-11-15T01:57:17.846Z+ gunzip
772024-11-15T01:57:20.090Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-11-15T01:57:20.092Z+ for t in "$input/bins/"*.gz
792024-11-15T01:57:20.095Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-11-15T01:57:20.095Z+ b=crucible-hammer.gz
812024-11-15T01:57:20.095Z+ b=crucible-hammer
822024-11-15T01:57:20.095Z+ gunzip
832024-11-15T01:57:22.252Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-11-15T01:57:22.255Z+ for t in "$input/bins/"*.gz
852024-11-15T01:57:22.258Z++ basename /input/build/work/bins/crutest.gz
862024-11-15T01:57:22.258Z+ b=crutest.gz
872024-11-15T01:57:22.258Z+ b=crutest
882024-11-15T01:57:22.258Z+ gunzip
892024-11-15T01:57:24.558Z+ chmod +x /var/tmp/bins/crutest
902024-11-15T01:57:24.560Z+ for t in "$input/bins/"*.gz
912024-11-15T01:57:24.563Z++ basename /input/build/work/bins/dsc.gz
922024-11-15T01:57:24.563Z+ b=dsc.gz
932024-11-15T01:57:24.563Z+ b=dsc
942024-11-15T01:57:24.563Z+ gunzip
952024-11-15T01:57:25.947Z+ chmod +x /var/tmp/bins/dsc
962024-11-15T01:57:25.950Z+ export BINDIR=/var/tmp/bins
972024-11-15T01:57:25.950Z+ BINDIR=/var/tmp/bins
982024-11-15T01:57:25.950Z+ jobpid=1141
992024-11-15T01:57:25.952Z+ echo 'Setup debug logging'
1002024-11-15T01:57:25.952ZSetup debug logging
1012024-11-15T01:57:25.952Z+ mkdir /tmp/debug
1022024-11-15T01:57:25.952Z+ sleep 3600
1032024-11-15T01:57:25.952Z+ psrinfo -v
1042024-11-15T01:57:25.956Z+ df -h
1052024-11-15T01:57:25.964Z+ prstat -d d -mLc 1
1062024-11-15T01:57:25.966Z+ iostat -T d -xn 1
1072024-11-15T01:57:25.966Z+ mpstat -T d 1
1082024-11-15T01:57:25.969Z+ vmstat -T d -p 1
1092024-11-15T01:57:25.969Z+ pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d
1102024-11-15T01:57:25.969Z+ banner test_up_encrypted
1112024-11-15T01:57:25.969Z+ pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d
1122024-11-15T01:57:25.971Z
1132024-11-15T01:57:25.971Z ##### ###### #### ##### # # ##### ###### # #
1142024-11-15T01:57:25.971Z # # # # # # # # # ## #
1152024-11-15T01:57:25.971Z # ##### #### # # # # # ##### # # #
1162024-11-15T01:57:25.971Z # # # # # # ##### # # # #
1172024-11-15T01:57:25.971Z # # # # # # # # # # ##
1182024-11-15T01:57:25.971Z # ###### #### # ####### #### # ####### ###### # #
1192024-11-15T01:57:25.971Z
1202024-11-15T01:57:25.971Z+ ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted
1212024-11-15T01:57:25.982Z/input/build/work
1222024-11-15T01:57:25.994ZTurn off color for downstairs dump
1232024-11-15T01:57:26.017ZUpstairs using key: 3i79V7maEKL+8Unt/QmMgH36yPQ9DPLMLcgaLFWLrpc=
1242024-11-15T01:57:26.020Zdsc output goes to /tmp/test_up-build/dsc-out.txt
1252024-11-15T01:57:26.020ZCreating 3 downstairs regions
1262024-11-15T01:57:26.322ZStarting 3 downstairs
1272024-11-15T01:57:26.324Zdsc started at PID: 1187
1282024-11-15T01:57:31.494ZDisable automatic restart on all downstairs
1292024-11-15T01:57:31.546Z
1302024-11-15T01:57:31.546ZBegin tests, output goes to /tmp/test_up-build/test_up_out.txt
1312024-11-15T01:57:31.549ZRunning test: span
1322024-11-15T01:57:35.728ZCompleted test: span
1332024-11-15T01:57:40.733ZRunning test: big
1342024-11-15T01:57:45.089ZCompleted test: big
1352024-11-15T01:57:50.094ZRunning test: dep
1362024-11-15T01:58:41.211ZCompleted test: dep
1372024-11-15T01:58:46.217ZRunning test: balloon
1382024-11-15T01:59:26.035ZCompleted test: balloon
1392024-11-15T01:59:31.040ZRunning test: deactivate
1402024-11-15T02:00:25.732ZCompleted test: deactivate
1412024-11-15T02:00:30.738ZRunning hammer
1422024-11-15T02:00:43.525ZRun repair tests
1432024-11-15T02:00:43.529Z/var/tmp/bins/crutest fill -g 31 -q --key 3i79V7maEKL+8Unt/QmMgH36yPQ9DPLMLcgaLFWLrpc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file
1442024-11-15T02:00:43.588Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.749130898Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"task":"crutest"}
1452024-11-15T02:00:43.590Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.749704655Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"task":"crutest"}
1462024-11-15T02:00:43.590Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.751429709Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"task":"crutest"}
1472024-11-15T02:00:43.590Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.751635661Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224}
1482024-11-15T02:00:43.590Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"5698e843f5bfd164775ff7ff0d12625b59494d67\",\n git_commit_timestamp: \"2024-11-15T01:38:50.000000000Z\",\n git_branch: \"alan/global-state-from-dtrace\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.751676647Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224}
1492024-11-15T02:00:43.590Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.751700367Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224}
1502024-11-15T02:00:43.591Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.751716243Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224}
1512024-11-15T02:00:43.593Z{"msg":"Crucible 7e279f07-fd44-42ae-862f-56b637745f95 has session id: 982d8806-06c0-44ae-8d07-ed61186fbfe0","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.75221289Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1522024-11-15T02:00:43.593Z{"msg":"Upstairs opts: Upstairs UUID: 7e279f07-fd44-42ae-862f-56b637745f95, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.752268372Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1532024-11-15T02:00:43.593Z{"msg":"Crucible stats registered with UUID: 7e279f07-fd44-42ae-862f-56b637745f95","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.752294082Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1542024-11-15T02:00:43.624Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786196472Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"task":"crutest"}
1552024-11-15T02:00:43.628Z{"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786250263Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224}
1562024-11-15T02:00:43.628Z{"msg":"7e279f07-fd44-42ae-862f-56b637745f95 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.78629236Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1572024-11-15T02:00:43.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786323242Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"0","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1582024-11-15T02:00:43.628Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786364838Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"0","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1592024-11-15T02:00:43.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786401033Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"1","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1602024-11-15T02:00:43.628Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786430344Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"1","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1612024-11-15T02:00:43.628Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786481605Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"2","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1622024-11-15T02:00:43.628Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786514658Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"2","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1632024-11-15T02:00:43.628Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786542479Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1642024-11-15T02:00:43.628Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786572591Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1652024-11-15T02:00:43.628Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786598791Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1662024-11-15T02:00:43.628Z{"msg":"ds_connection connected from Ok(127.0.0.1:48898)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786623701Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1672024-11-15T02:00:43.628Z{"msg":"ds_connection connected from Ok(127.0.0.1:38532)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786649161Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1682024-11-15T02:00:43.628Z{"msg":"ds_connection connected from Ok(127.0.0.1:40164)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.786672891Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1692024-11-15T02:00:43.628Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.787487394Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"0","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1702024-11-15T02:00:43.628Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.787539874Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"0","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1712024-11-15T02:00:43.628Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.787611053Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"2","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1722024-11-15T02:00:43.628Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.787643245Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"2","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1732024-11-15T02:00:43.628Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.787674848Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"1","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1742024-11-15T02:00:43.629Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.787699418Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"1","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1752024-11-15T02:00:43.629Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788175436Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"0","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1762024-11-15T02:00:43.629Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788216673Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"1","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1772024-11-15T02:00:43.629Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.78823523Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"2","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1782024-11-15T02:00:43.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788590431Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"1","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1792024-11-15T02:00:43.629Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.78863903Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"1","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1802024-11-15T02:00:43.629Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788717401Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1812024-11-15T02:00:43.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788747963Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"2","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1822024-11-15T02:00:43.629Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.78876414Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"2","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1832024-11-15T02:00:43.629Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788787289Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1842024-11-15T02:00:43.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788800754Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"0","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1852024-11-15T02:00:43.629Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.7888147Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"0","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1862024-11-15T02:00:43.629Z{"msg":"[0]R flush_numbers: [1495, 1495, 1495, 1495, 1495]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788835778Z","hostname":Wait for a query_work_queue command to finish before sending IO
1872024-11-15T02:00:43.629Z"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1882024-11-15T02:00:43.629Z{"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788859708Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1892024-11-15T02:00:43.629Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788880226Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1902024-11-15T02:00:43.629Z{"msg":"[1]R flush_numbers: [1495, 1495, 1495, 1495, 1495]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788894441Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1912024-11-15T02:00:43.629Z{"msg":"[1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788909177Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1922024-11-15T02:00:43.629Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788922843Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1932024-11-15T02:00:43.629Z{"msg":"[2]R flush_numbers: [1495, 1495, 1495, 1495, 1495]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788944031Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1942024-11-15T02:00:43.629Z{"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788958217Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1952024-11-15T02:00:43.629Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.788972172Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1962024-11-15T02:00:43.629Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
1972024-11-15T02:00:43.629Z,"time":"2024-11-15T02:00:42.789003324Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
1982024-11-15T02:00:43.629Z{"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789033516Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"Fill test
1992024-11-15T02:00:43.629Z982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2002024-11-15T02:00:43.632Z{"msg":"Next flush: 1496","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789051544Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2012024-11-15T02:00:43.632Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789072262Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"":"downstairs","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2022024-11-15T02:00:43.632Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789086237Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2032024-11-15T02:00:43.632Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789098472Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"0","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2042024-11-15T02:00:43.632Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789119421Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"1","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2052024-11-15T02:00:43.632Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789133236Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"client":"2","":"downstairs","session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2062024-11-15T02:00:43.632Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789147181Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2072024-11-15T02:00:43.632Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789159517Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2082024-11-15T02:00:43.632Z{"msg":"7e279f07-fd44-42ae-862f-56b637745f95 is now active with session: 982d8806-06c0-44ae-8d07-ed61186fbfe0","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789173102Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2092024-11-15T02:00:43.632Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789185297Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224,"session_id":"982d8806-06c0-44ae-8d07-ed61186fbfe0"}
2102024-11-15T02:00:43.632Z{"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:42.789205095Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1224}
2112024-11-15T02:00:43.665ZRead and Verify all blocks (0..50 range:false)
2122024-11-15T02:00:43.703ZWrote out file "/var/tmp/test_up-build/verify_file"
2132024-11-15T02:00:43.703ZCLIENT: Tests done. All submitted work has been ACK'd
2142024-11-15T02:00:43.706Z----------------------------------------------------------------
2152024-11-15T02:00:43.706Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1
2162024-11-15T02:00:43.706ZStates: Active Active Active
2172024-11-15T02:00:43.706ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
2182024-11-15T02:00:43.706Z 1002 Acked Read 50 Done Done Done false
2192024-11-15T02:00:43.706Z STATES DS:0 DS:1 DS:2 TOTAL
2202024-11-15T02:00:43.706Z Sent 0 0 0 0
2212024-11-15T02:00:43.706Z Done 1 1 1 3
2222024-11-15T02:00:43.706Z Skipped 0 0 0 0
2232024-11-15T02:00:43.706Z Error 0 0 0 0
2242024-11-15T02:00:43.706ZLast Flush: 1001 1001 1001
2252024-11-15T02:00:43.706ZDownstairs last five completed: 1001 1000
2262024-11-15T02:00:43.706ZUpstairs last five completed: 1002 1001 1000
2272024-11-15T02:00:43.706ZCLIENT: Up:0 ds:1 act:3
2282024-11-15T02:00:47.704Z----------------------------------------------------------------
2292024-11-15T02:00:47.704Z Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0
2302024-11-15T02:00:47.704ZDownstairs last five completed: 1003 1002 1001 1000
2312024-11-15T02:00:47.704ZUpstairs last five completed: 1003 1002 1001 1000
2322024-11-15T02:00:47.704ZCLIENT: Up:0 ds:0 act:3
2332024-11-15T02:00:47.704ZCLIENT: All crucible jobs finished, exiting program
2342024-11-15T02:00:47.716ZRepair setup passed
2352024-11-15T02:00:47.719ZCopy the region for /var/tmp/test_up-build/8830
2362024-11-15T02:00:47.721Zcp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous
2372024-11-15T02:00:47.724Z/var/tmp/bins/crutest fill -g 32 -q --key 3i79V7maEKL+8Unt/QmMgH36yPQ9DPLMLcgaLFWLrpc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
2382024-11-15T02:00:47.781Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.943849045Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"task":"crutest"}
2392024-11-15T02:00:47.783Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.944376734Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"task":"crutest"}
2402024-11-15T02:00:47.783Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.946124067Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"task":"crutest"}
2412024-11-15T02:00:47.783Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.94633376Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230}
2422024-11-15T02:00:47.783Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"5698e843f5bfd164775ff7ff0d12625b59494d67\",\n git_commit_timestamp: \"2024-11-15T01:38:50.000000000Z\",\n git_branch: \"alan/global-state-from-dtrace\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.946381829Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230}
2432024-11-15T02:00:47.783Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.946406559Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230}
2442024-11-15T02:00:47.783Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.946422515Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230}
2452024-11-15T02:00:47.786Z{"msg":"Crucible 2417264f-e2f9-494c-922c-26cc5a169d60 has session id: bc04a1da-a7fa-436c-a077-f63dfba80649","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.947001494Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2462024-11-15T02:00:47.786Z{"msg":"Upstairs opts: Upstairs UUID: 2417264f-e2f9-494c-922c-26cc5a169d60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.947054776Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2472024-11-15T02:00:47.786Z{"msg":"Crucible stats registered with UUID: 2417264f-e2f9-494c-922c-26cc5a169d60","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.947081366Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2482024-11-15T02:00:47.819Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.981763535Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"task":"crutest"}
2492024-11-15T02:00:47.819Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.98182568Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230}
2502024-11-15T02:00:47.822Z{"msg":"2417264f-e2f9-494c-922c-26cc5a169d60 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.98187634Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2512024-11-15T02:00:47.822Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.981913735Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"0","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2522024-11-15T02:00:47.822Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.98197825Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"0","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2532024-11-15T02:00:47.822Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.982035253Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"1","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2542024-11-15T02:00:47.822Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.982106741Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"1","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2552024-11-15T02:00:47.822Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.982160033Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"2","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2562024-11-15T02:00:47.822Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.982208362Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"2","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2572024-11-15T02:00:47.822Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.982447887Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2582024-11-15T02:00:47.822Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.982484491Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2592024-11-15T02:00:47.822Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.982513413Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2602024-11-15T02:00:47.822Z{"msg":"ds_connection connected from Ok(127.0.0.1:49098)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.982550317Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2612024-11-15T02:00:47.822Z{"msg":"ds_connection connected from Ok(127.0.0.1:46790)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.98258369Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2622024-11-15T02:00:47.822Z{"msg":"ds_connection connected from Ok(127.0.0.1:57818)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.982612262Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2632024-11-15T02:00:47.822Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.983013781Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"0","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2642024-11-15T02:00:47.822Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.983053497Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"0","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2652024-11-15T02:00:47.822Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.983072284Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"2","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2662024-11-15T02:00:47.822Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.983105767Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"2","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2672024-11-15T02:00:47.822Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.983126946Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"1","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2682024-11-15T02:00:47.822Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.983153216Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"1","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2692024-11-15T02:00:47.822Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.98374389Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"2","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2702024-11-15T02:00:47.822Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.983802424Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"0","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2712024-11-15T02:00:47.822Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.983835417Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"1","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2722024-11-15T02:00:47.822Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984140618Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"2","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2732024-11-15T02:00:47.822Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984179843Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"2","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2742024-11-15T02:00:47.822Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984201082Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2752024-11-15T02:00:47.822Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984270189Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"1","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2762024-11-15T02:00:47.822Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984311565Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"1","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2772024-11-15T02:00:47.822Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984339286Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2782024-11-15T02:00:47.822Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984370459Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"0","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2792024-11-15T02:00:47.822Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984397589Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"0","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2802024-11-15T02:00:47.822Z{"msg":"[0]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984433144Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2812024-11-15T02:00:47.822Z{"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984461285Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2822024-11-15T02:00:47.822Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO
2832024-11-15T02:00:47.823Z:"2024-11-15T02:00:46.984482783Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2842024-11-15T02:00:47.825Z{"msg":"[1]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984546579Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2852024-11-15T02:00:47.825Z{"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984582733Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2862024-11-15T02:00:47.825Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984610354Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2872024-11-15T02:00:47.826Z{"msg":"[2]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984642086Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2882024-11-15T02:00:47.826Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"Region: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
2892024-11-15T02:00:47.826Zcrucible","level":30,"time":"2024-11-15T02:00:46.984672799Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2902024-11-15T02:00:47.826Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984705982Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2912024-11-15T02:00:47.826Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984732382Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2922024-11-15T02:00:47.826Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984763775Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2932024-11-15T02:00:47.826Z{"msg":"Next flush: 1497","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984789235Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2942024-11-15T02:00:47.826Z{"msg":"All extents match","v":0,"name":"crucible","level":30Loading write count information from file "/var/tmp/test_up-build/verify_file"
2952024-11-15T02:00:47.826Z,"time":"2024-11-15T02:00:46.984823258Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"":"downstairs","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2962024-11-15T02:00:47.826Z{"msg":"Fill test
2972024-11-15T02:00:47.826ZNo downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984871237Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2982024-11-15T02:00:47.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984896187Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"0","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
2992024-11-15T02:00:47.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984924669Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"1","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
3002024-11-15T02:00:47.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984949799Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"client":"2","":"downstairs","session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
3012024-11-15T02:00:47.826Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.984985493Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
3022024-11-15T02:00:47.826Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.985009622Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
3032024-11-15T02:00:47.826Z{"msg":"2417264f-e2f9-494c-922c-26cc5a169d60 is now active with session: bc04a1da-a7fa-436c-a077-f63dfba80649","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.985035782Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
3042024-11-15T02:00:47.826Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.985050058Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230,"session_id":"bc04a1da-a7fa-436c-a077-f63dfba80649"}
3052024-11-15T02:00:47.826Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:46.985071027Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230}
3062024-11-15T02:00:47.859ZRead and Verify all blocks (0..50 range:false)
3072024-11-15T02:00:47.897ZWrote out file "/var/tmp/test_up-build/verify_file"
3082024-11-15T02:00:47.897ZCLIENT: Tests done. All submitted work has been ACK'd
3092024-11-15T02:00:47.899Z----------------------------------------------------------------
3102024-11-15T02:00:47.899Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:1
3112024-11-15T02:00:47.899ZStates: Active Active Active
3122024-11-15T02:00:47.899ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
3132024-11-15T02:00:47.899Z 1002 Acked Read 50 Done Done Done false
3142024-11-15T02:00:47.899Z STATES DS:0 DS:1 DS:2 TOTAL
3152024-11-15T02:00:47.899Z Sent 0 0 0 0
3162024-11-15T02:00:47.899Z Done 1 1 1 3
3172024-11-15T02:00:47.900Z Skipped 0 0 0 0
3182024-11-15T02:00:47.900Z Error 0 0 0 0
3192024-11-15T02:00:47.900ZLast Flush: 1001 1001 1001
3202024-11-15T02:00:47.900ZDownstairs last five completed: 1001 1000
3212024-11-15T02:00:47.900ZUpstairs last five completed: 1002 1001 1000
3222024-11-15T02:00:47.900ZCLIENT: Up:0 ds:1 act:3
3232024-11-15T02:00:51.898Z----------------------------------------------------------------
3242024-11-15T02:00:51.898Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:0
3252024-11-15T02:00:51.898ZDownstairs last five completed: 1003 1002 1001 1000
3262024-11-15T02:00:51.898ZUpstairs last five completed: 1003 1002 1001 1000
3272024-11-15T02:00:51.898ZCLIENT: Up:0 ds:0 act:3
3282024-11-15T02:00:51.898ZCLIENT: All crucible jobs finished, exiting program
3292024-11-15T02:00:51.900Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-15T02:00:51.062365482Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1230}
3302024-11-15T02:00:51.905ZRepair part 1 passed
3312024-11-15T02:00:51.907Z
3322024-11-15T02:00:51.907ZKill the current downstairs
3332024-11-15T02:00:52.067ZDownstairs 2 stopped
3342024-11-15T02:00:52.067Zmv /var/tmp/test_up-build/8830 /var/tmp/test_up-build/new
3352024-11-15T02:00:52.070ZNow put back the original so we have a mismatch
3362024-11-15T02:00:52.070Zmv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830
3372024-11-15T02:00:52.072ZRestart downstairs with old directory
3382024-11-15T02:00:52.123Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
3392024-11-15T02:00:52.139ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3402024-11-15T02:00:52.141Z 0 000-009 32 32 31 1497 1497 1496 F F F <---
3412024-11-15T02:00:52.141Z 1 010-019 32 32 31 1497 1497 1496 F F F <---
3422024-11-15T02:00:52.141Z 2 020-029 32 32 31 1497 1497 1496 F F F <---
3432024-11-15T02:00:52.141Z 3 030-039 32 32 31 1497 1497 1496 F F F <---
3442024-11-15T02:00:52.141Z 4 040-049 32 32 31 1497 1497 1496 F F F <---
3452024-11-15T02:00:52.141ZMax gen: 32, Max flush: 1497
3462024-11-15T02:00:52.141ZError: Difference in extent metadata found!
3472024-11-15T02:00:52.144Zdump test found error as expected
3482024-11-15T02:00:52.144Z
3492024-11-15T02:00:52.144Z
3502024-11-15T02:00:52.144Z/var/tmp/bins/crutest verify --range -g 33 -q --key 3i79V7maEKL+8Unt/QmMgH36yPQ9DPLMLcgaLFWLrpc= --dsc 127.0.0.1:9998 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file
3512024-11-15T02:00:52.200Z{"msg":"Use this region info from dsc: RegionExtentInfo { block_size: 4096, blocks_per_extent: 10, extent_count: 5 }","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.364739334Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"task":"crutest"}
3522024-11-15T02:00:52.203Z{"msg":"dsc has 3 regions. This means 1 sub_volumes","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.365279548Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"task":"crutest"}
3532024-11-15T02:00:52.203Z{"msg":"SV 0 has targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.367005603Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"task":"crutest"}
3542024-11-15T02:00:52.203Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.367207743Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238}
3552024-11-15T02:00:52.203Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"5698e843f5bfd164775ff7ff0d12625b59494d67\",\n git_commit_timestamp: \"2024-11-15T01:38:50.000000000Z\",\n git_branch: \"alan/global-state-from-dtrace\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.367246128Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238}
3562024-11-15T02:00:52.203Z{"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.367262974Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238}
3572024-11-15T02:00:52.203Z{"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.367297398Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238}
3582024-11-15T02:00:52.205Z{"msg":"Crucible 15c8834d-9d20-411a-8c91-5b14794bb95a has session id: 560c57b8-42e8-49e5-b4d1-ce98a5e96fc4","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.367855439Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3592024-11-15T02:00:52.205Z{"msg":"Upstairs opts: Upstairs UUID: 15c8834d-9d20-411a-8c91-5b14794bb95a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.367900667Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3602024-11-15T02:00:52.205Z{"msg":"Crucible stats registered with UUID: 15c8834d-9d20-411a-8c91-5b14794bb95a","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.367919645Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3612024-11-15T02:00:52.237Z{"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.401971404Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"task":"crutest"}
3622024-11-15T02:00:52.237Z{"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402027907Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238}
3632024-11-15T02:00:52.240Z{"msg":"15c8834d-9d20-411a-8c91-5b14794bb95a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402099185Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3642024-11-15T02:00:52.240Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402129927Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3652024-11-15T02:00:52.240Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402165661Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3662024-11-15T02:00:52.240Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402215441Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3672024-11-15T02:00:52.240Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402252566Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3682024-11-15T02:00:52.240Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402285138Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3692024-11-15T02:00:52.240Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402327565Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3702024-11-15T02:00:52.240Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402348774Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3712024-11-15T02:00:52.240Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402391991Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3722024-11-15T02:00:52.240Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402435748Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3732024-11-15T02:00:52.240Z{"msg":"ds_connection connected from Ok(127.0.0.1:60588)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402467791Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3742024-11-15T02:00:52.240Z{"msg":"ds_connection connected from Ok(127.0.0.1:46382)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.402518711Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3752024-11-15T02:00:52.240Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-11-15T02:00:51.402557626Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3762024-11-15T02:00:52.240Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-11-15T02:00:51.402607096Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3772024-11-15T02:00:52.240Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-11-15T02:00:51.402649593Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3782024-11-15T02:00:52.240Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.40266833Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3792024-11-15T02:00:52.240Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.403128763Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3802024-11-15T02:00:52.240Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.403166188Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3812024-11-15T02:00:52.240Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.403182564Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3822024-11-15T02:00:52.241Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.403203883Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3832024-11-15T02:00:52.241Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.40370088Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3842024-11-15T02:00:52.241Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.403837054Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3852024-11-15T02:00:52.241Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.404017245Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3862024-11-15T02:00:52.241Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.404058592Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3872024-11-15T02:00:52.241Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.404088343Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3882024-11-15T02:00:52.241Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.404120376Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3892024-11-15T02:00:52.241Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.4041554Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3902024-11-15T02:00:52.241Z{"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:00:51.404189313Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3912024-11-15T02:01:02.236Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.403549369Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3922024-11-15T02:01:02.236Z{"msg":"ds_connection connected from Ok(127.0.0.1:37334)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.403639775Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3932024-11-15T02:01:02.241Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.40476217Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3942024-11-15T02:01:02.241Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.40481424Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3952024-11-15T02:01:02.241Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405384396Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3962024-11-15T02:01:02.241Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405719549Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3972024-11-15T02:01:02.241Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405750742Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3982024-11-15T02:01:02.241Z{"msg":"[0]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405769129Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
3992024-11-15T02:01:02.241Z{"msg":"[0]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405785105Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4002024-11-15T02:01:02.241Z{"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405800181Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4012024-11-15T02:01:02.242Z{"msg":"[1]R flush_numbers: [1497, 1497, 1497, 1497, 1497]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.40582256Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4022024-11-15T02:01:02.242Z{"msg":"[1]R generation: [32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405837326Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4032024-11-15T02:01:02.242Z{"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405858214Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4042024-11-15T02:01:02.242Z{"msg":"[2]R flush_numbers: [1496, 1496, 1496, 1496, 1496]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.40587254Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4052024-11-15T02:01:02.242Z{"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405893418Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4062024-11-15T02:01:02.242Z{"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405907894Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4072024-11-15T02:01:02.242Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405928192Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4082024-11-15T02:01:02.242Z{"msg":"Generation requested: 33 >= found:33","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405942298Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4092024-11-15T02:01:02.242Z{"msg":"Next flush: 1498","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405962626Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4102024-11-15T02:01:02.242Z{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.405976631Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4112024-11-15T02:01:02.242Z{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.40599794Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4122024-11-15T02:01:02.242Z{"msg":"extent:0 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406013576Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4132024-11-15T02:01:02.242Z{"msg":"extent:0 flush: 1497 1497 1496 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406029072Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4142024-11-15T02:01:02.242Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406044038Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4152024-11-15T02:01:02.242Z{"msg":"extent:0 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406065546Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4162024-11-15T02:01:02.242Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406081302Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4172024-11-15T02:01:02.242Z{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406103291Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4182024-11-15T02:01:02.242Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406118607Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4192024-11-15T02:01:02.242Z{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406145928Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4202024-11-15T02:01:02.242Z{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406169287Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4212024-11-15T02:01:02.242Z{"msg":"extent:1 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406185054Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4222024-11-15T02:01:02.242Z{"msg":"extent:1 flush: 1497 1497 1496 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406207102Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4232024-11-15T02:01:02.242Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406222469Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4242024-11-15T02:01:02.242Z{"msg":"extent:1 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406257052Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4252024-11-15T02:01:02.242Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406281552Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4262024-11-15T02:01:02.242Z{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406297048Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4272024-11-15T02:01:02.242Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406318797Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4282024-11-15T02:01:02.242Z{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406335723Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4292024-11-15T02:01:02.242Z{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406356932Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4302024-11-15T02:01:02.242Z{"msg":"extent:2 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406372108Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4312024-11-15T02:01:02.242Z{"msg":"extent:2 flush: 1497 1497 1496 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406393826Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4322024-11-15T02:01:02.242Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406409112Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4332024-11-15T02:01:02.242Z{"msg":"extent:2 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406430561Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4342024-11-15T02:01:02.242Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406445747Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4352024-11-15T02:01:02.242Z{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406467226Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4362024-11-15T02:01:02.242Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406482301Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4372024-11-15T02:01:02.242Z{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.40650511Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4382024-11-15T02:01:02.242Z{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406521037Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4392024-11-15T02:01:02.243Z{"msg":"extent:3 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406542615Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4402024-11-15T02:01:02.243Z{"msg":"extent:3 flush: 1497 1497 1496 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406557902Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4412024-11-15T02:01:02.243Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406573297Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4422024-11-15T02:01:02.243Z{"msg":"extent:3 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406589024Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4432024-11-15T02:01:02.243Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406611203Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4442024-11-15T02:01:02.243Z{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406626289Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4452024-11-15T02:01:02.243Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406641575Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4462024-11-15T02:01:02.243Z{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406662233Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4472024-11-15T02:01:02.243Z{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406688013Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4482024-11-15T02:01:02.243Z{"msg":"extent:4 gens: 32 32 31","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406727729Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4492024-11-15T02:01:02.243Z{"msg":"extent:4 flush: 1497 1497 1496 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406765244Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4502024-11-15T02:01:02.243Z{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406802899Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4512024-11-15T02:01:02.243Z{"msg":"extent:4 dirty: false false false","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406837142Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4522024-11-15T02:01:02.243Z{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406873987Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4532024-11-15T02:01:02.243Z{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406909951Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4542024-11-15T02:01:02.243Z{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406945665Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"mrl":"flush_mismatch","":"mend","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4552024-11-15T02:01:02.243Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.406973486Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4562024-11-15T02:01:02.243Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.407005279Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4572024-11-15T02:01:02.243Z{"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.407024937Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4582024-11-15T02:01:02.243Z{"msg":"starting reconciliation a2d855d5-a90d-4ed0-8366-6103eadc9aab: found 5 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.40705788Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4592024-11-15T02:01:02.243Z{"msg":"Full repair list: {ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(3): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.407119864Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4602024-11-15T02:01:02.243Z{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(3), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(3), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(3) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1498, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.407240062Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4612024-11-15T02:01:02.243Z{"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.407274005Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4622024-11-15T02:01:02.243Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.407303087Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4632024-11-15T02:01:02.243Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.407330968Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4642024-11-15T02:01:02.246Z{"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.40743732Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4652024-11-15T02:01:02.246Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.407472634Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4662024-11-15T02:01:02.249Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.414933232Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4672024-11-15T02:01:02.249Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.41533212Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4682024-11-15T02:01:02.249Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.415580268Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4692024-11-15T02:01:02.249Z{"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.415856958Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4702024-11-15T02:01:02.249Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.415927636Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4712024-11-15T02:01:02.252Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.418942691Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4722024-11-15T02:01:02.252Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.41926838Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4732024-11-15T02:01:02.252Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.419531625Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4742024-11-15T02:01:02.252Z{"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.419803953Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4752024-11-15T02:01:02.254Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.419851461Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4762024-11-15T02:01:02.257Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.422938395Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4772024-11-15T02:01:02.257Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.4233294Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4782024-11-15T02:01:02.257Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.423583771Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4792024-11-15T02:01:02.257Z{"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.423861781Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4802024-11-15T02:01:02.257Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.423905929Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4812024-11-15T02:01:02.259Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.426696465Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4822024-11-15T02:01:02.259Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.427037761Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4832024-11-15T02:01:02.261Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.427331647Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4842024-11-15T02:01:02.261Z{"msg":"sending reconcile request ReconciliationId(18)","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.427661648Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4852024-11-15T02:01:02.261Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.427699663Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4862024-11-15T02:01:02.264Z{"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.430539138Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4872024-11-15T02:01:02.264Z{"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.430866919Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4882024-11-15T02:01:02.264Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.430917879Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"0","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4892024-11-15T02:01:02.264Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.430958915Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"1","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4902024-11-15T02:01:02.264Z{"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.430999761Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"client":"2","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4912024-11-15T02:01:02.264Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.431030063Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4922024-11-15T02:01:02.264Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.431056354Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4932024-11-15T02:01:02.264Z{Wait for a query_work_queue command to finish before sending IO
4942024-11-15T02:01:02.264Z"msg":"15c8834d-9d20-411a-8c91-5b14794bb95a is now active with session: 560c57b8-42e8-49e5-b4d1-ce98a5e96fc4","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.431108394Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4952024-11-15T02:01:02.264Z{"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-11-15T02:01:01.431124301Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238}
4962024-11-15T02:01:02.264Z{"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-11-15T02:01:01.431137106Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238,"reconcile":"a2d855d5-a90d-4ed0-8366-6103eadc9aab","":"downstairs","":"downstairs","session_id":"560c57b8-42e8-49e5-b4d1-ce98a5e96fc4"}
4972024-11-15T02:01:02.264ZRegion: sv:1 bs:4096 ts:204800 tb:50 max_io:50 or 204800
4982024-11-15T02:01:02.264ZLoading write count information from file "/var/tmp/test_up-build/verify_file"
4992024-11-15T02:01:02.264ZRead and Verify all blocks (0..50 range:true)
5002024-11-15T02:01:02.303ZWrote out file "/var/tmp/test_up-build/verify_file"
5012024-11-15T02:01:02.303ZVerify test completed
5022024-11-15T02:01:02.305ZWrote out file "/var/tmp/test_up-build/verify_file"
5032024-11-15T02:01:02.305ZCLIENT: Tests done. All submitted work has been ACK'd
5042024-11-15T02:01:02.305Z----------------------------------------------------------------
5052024-11-15T02:01:02.305Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1
5062024-11-15T02:01:02.305ZStates: Active Active Active
5072024-11-15T02:01:02.305ZJOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY
5082024-11-15T02:01:02.305Z 1000 Acked Read 50 Done Done Done false
5092024-11-15T02:01:02.305Z STATES DS:0 DS:1 DS:2 TOTAL
5102024-11-15T02:01:02.305Z Sent 0 0 0 0
5112024-11-15T02:01:02.305Z Done 1 1 1 3
5122024-11-15T02:01:02.305Z Skipped 0 0 0 0
5132024-11-15T02:01:02.305Z Error 0 0 0 0
5142024-11-15T02:01:02.305ZLast Flush: 0 0 0
5152024-11-15T02:01:02.306ZDownstairs last five completed:
5162024-11-15T02:01:02.306ZUpstairs last five completed: 1000
5172024-11-15T02:01:02.306ZCLIENT: Up:0 ds:1 act:3
5182024-11-15T02:01:06.303Z----------------------------------------------------------------
5192024-11-15T02:01:06.304Z Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0
5202024-11-15T02:01:06.304ZDownstairs last five completed: 1001 1000
5212024-11-15T02:01:06.304ZUpstairs last five completed: 1001 1000
5222024-11-15T02:01:06.304ZCLIENT: Up:0 ds:0 act:3
5232024-11-15T02:01:06.304ZCLIENT: All crucible jobs finished, exiting program
5242024-11-15T02:01:06.306Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-15T02:01:05.473312939Z","hostname":"w-01JCPRHS4XZARP6XNF1VK212KM","pid":1238}
5252024-11-15T02:01:06.311ZRepair part 2 passed
5262024-11-15T02:01:06.313Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830
5272024-11-15T02:01:06.328ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5282024-11-15T02:01:06.328Z 0 000-009 32 32 32 1497 1497 1497 F F F
5292024-11-15T02:01:06.330Z 1 010-019 32 32 32 1497 1497 1497 F F F
5302024-11-15T02:01:06.330Z 2 020-029 32 32 32 1497 1497 1497 F F F
5312024-11-15T02:01:06.330Z 3 030-039 32 32 32 1497 1497 1497 F F F
5322024-11-15T02:01:06.330Z 4 040-049 32 32 32 1497 1497 1497 F F F
5332024-11-15T02:01:06.330ZMax gen: 32, Max flush: 1497
5342024-11-15T02:01:06.333Zdump test passed
5352024-11-15T02:01:06.333Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1
5362024-11-15T02:01:06.346Z Extent 1
5372024-11-15T02:01:06.346ZGEN 32 32 32
5382024-11-15T02:01:06.346ZFLUSH_ID 1497 1497 1497
5392024-11-15T02:01:06.346ZDIRTY
5402024-11-15T02:01:06.347Z
5412024-11-15T02:01:06.347ZBLOCK D0 D1 D2 C0 C1 C2 DIFF
5422024-11-15T02:01:06.349Z010 A A A A A A
5432024-11-15T02:01:06.349Z011 A A A A A A
5442024-11-15T02:01:06.352Z012 A A A A A A
5452024-11-15T02:01:06.352Z013 A A A A A A
5462024-11-15T02:01:06.354Z014 A A A A A A
5472024-11-15T02:01:06.356Z015 A A A A A A
5482024-11-15T02:01:06.356Z016 A A A A A A
5492024-11-15T02:01:06.358Z017 A A A A A A
5502024-11-15T02:01:06.359Z018 A A A A A A
5512024-11-15T02:01:06.361Z019 A A A A A A
5522024-11-15T02:01:06.363Zdump extent test passed
5532024-11-15T02:01:06.363Z/var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20
5542024-11-15T02:01:06.376ZExtent 2 Block in extent 0 Actual block 20
5552024-11-15T02:01:06.376Z
5562024-11-15T02:01:06.379Z DATA SHA256 VER
5572024-11-15T02:01:06.379Z------ ---------------------------------------------------------------- ---
5582024-11-15T02:01:06.379Z 0 d2d0382dfd3de8346ee101bfa12c0e01b4b457ca975ec6ba9dd31d78e2aafb3d A
5592024-11-15T02:01:06.379Z 1 d2d0382dfd3de8346ee101bfa12c0e01b4b457ca975ec6ba9dd31d78e2aafb3d A
5602024-11-15T02:01:06.379Z 2 d2d0382dfd3de8346ee101bfa12c0e01b4b457ca975ec6ba9dd31d78e2aafb3d A
5612024-11-15T02:01:06.379Z
5622024-11-15T02:01:06.379ZNONCES 0 1 2 DIFF
5632024-11-15T02:01:06.379Z------ ------------------------ ------------------------ ------------------------ -----
5642024-11-15T02:01:06.379Z 0 3872e368efed7d0899d8c328 3872e368efed7d0899d8c328 3872e368efed7d0899d8c328
5652024-11-15T02:01:06.379Z
5662024-11-15T02:01:06.379Z TAGS 0 1 2 DIFF
5672024-11-15T02:01:06.379Z------ -------------------------------- -------------------------------- -------------------------------- -----
5682024-11-15T02:01:06.379Z 0 0b1c05cd10da8161938434856072298a 0b1c05cd10da8161938434856072298a 0b1c05cd10da8161938434856072298a
5692024-11-15T02:01:06.379Z
5702024-11-15T02:01:06.379ZHASHES 0 1 2 DIFF
5712024-11-15T02:01:06.379Z------ ---------------- ---------------- ---------------- -----
5722024-11-15T02:01:06.379Z
5732024-11-15T02:01:06.382Zdump block test passed
5742024-11-15T02:01:06.382ZInitial upstairs tests have completed, stopping all downstairs
5752024-11-15T02:01:11.443ZCreating 4 larger downstairs regions
5762024-11-15T02:01:11.551ZStarting 4 downstairs
5772024-11-15T02:01:16.557Zdsc restarted at PID: 1251
5782024-11-15T02:01:16.773ZNow do the replace-reconcile test
5792024-11-15T02:01:16.825ZUsing 8840 for the replacement port
5802024-11-15T02:04:07.910ZCompleted test: replace-reconcile
5812024-11-15T02:04:07.915ZNow do the replace-before-active test
5822024-11-15T02:07:14.318ZCompleted test: replace-before-active
5832024-11-15T02:07:14.320ZAll tests have completed, stopping all downstairs
5842024-11-15T02:07:14.375Z
5852024-11-15T02:07:14.375ZAll Tests have passed
5862024-11-15T02:07:14.378Z9:48 Test duration
5872024-11-15T02:07:14.378Z
5882024-11-15T02:07:14.378Zreal 9:48.605881633
5892024-11-15T02:07:14.378Zuser 6:19.757334287
5902024-11-15T02:07:14.378Zsys 11.760716383
5912024-11-15T02:07:14.378Ztrap 0.191294509
5922024-11-15T02:07:14.378Ztflt 0.016581645
5932024-11-15T02:07:14.378Zdflt 0.031843010
5942024-11-15T02:07:14.378Zkflt 0.001266855
5952024-11-15T02:07:14.378Zlock 8:21:45.670078646
5962024-11-15T02:07:14.378Zslp 33:32.780077305
5972024-11-15T02:07:14.378Zlat 24.433618163
5982024-11-15T02:07:14.378Zstop 0.131183268
5992024-11-15T02:07:14.378Z+ echo 'test-up-encrypted ends'
6002024-11-15T02:07:14.378Ztest-up-encrypted ends
6012024-11-15T02:07:19.379Zprocess exited: duration 596764 ms, exit code 0
6022024-11-15T02:07:19.380Zexec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close
6032024-11-15T02:08:19.404Zexec warning: : stdout descriptor may be held open by a background process; giving up!
6042024-11-15T02:08:19.404Zexec warning: : stderr descriptor may be held open by a background process; giving up!
 
6052024-11-15T02:08:19.417Zfound 14 output files
6062024-11-15T02:08:19.417Zuploading: /tmp/test_up-build/dsc-out.txt (39496 bytes)
6072024-11-15T02:08:21.429Zuploaded: /tmp/test_up-build/dsc-out.txt
6082024-11-15T02:08:21.429Zuploading: /tmp/test_up-build/test_up_out.txt (10378915 bytes)
6092024-11-15T02:08:22.636Zuploaded: /tmp/test_up-build/test_up_out.txt
6102024-11-15T02:08:22.639Zuploading: /tmp/test_up-build/dsc/downstairs-8810.txt (2505 bytes)
6112024-11-15T02:08:23.646Zuploaded: /tmp/test_up-build/dsc/downstairs-8810.txt
6122024-11-15T02:08:23.646Zuploading: /tmp/test_up-build/dsc/downstairs-8820.txt (144634 bytes)
6132024-11-15T02:08:24.663Zuploaded: /tmp/test_up-build/dsc/downstairs-8820.txt
6142024-11-15T02:08:24.664Zuploading: /tmp/test_up-build/dsc/downstairs-8830.txt (251103 bytes)
6152024-11-15T02:08:25.688Zuploaded: /tmp/test_up-build/dsc/downstairs-8830.txt
6162024-11-15T02:08:25.691Zuploading: /tmp/test_up-build/dsc/downstairs-8840.txt (139443 bytes)
6172024-11-15T02:08:26.707Zuploaded: /tmp/test_up-build/dsc/downstairs-8840.txt
6182024-11-15T02:08:26.709Zuploading: /tmp/debug/df.txt (1270 bytes)
6192024-11-15T02:08:27.715Zuploaded: /tmp/debug/df.txt
6202024-11-15T02:08:27.717Zuploading: /tmp/debug/dtrace.txt (476933 bytes)
6212024-11-15T02:08:27.744Zupload warning: file "/tmp/debug/dtrace.txt" changed size mid upload: 476933 -> 563303
6222024-11-15T02:08:28.750Zuploaded: /tmp/debug/dtrace.txt
6232024-11-15T02:08:28.750Zuploading: /tmp/debug/iostat.txt (196889 bytes)
6242024-11-15T02:08:28.767Zupload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 196889 -> 199899
6252024-11-15T02:08:29.773Zuploaded: /tmp/debug/iostat.txt
6262024-11-15T02:08:29.775Zuploading: /tmp/debug/mpstat.txt (477427 bytes)
6272024-11-15T02:08:29.800Zupload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 477427 -> 485457
6282024-11-15T02:08:30.805Zuploaded: /tmp/debug/mpstat.txt
6292024-11-15T02:08:30.808Zuploading: /tmp/debug/paging.txt (86292 bytes)
6302024-11-15T02:08:30.816Zupload warning: file "/tmp/debug/paging.txt" changed size mid upload: 86292 -> 87842
6312024-11-15T02:08:31.822Zuploaded: /tmp/debug/paging.txt
6322024-11-15T02:08:31.826Zuploading: /tmp/debug/prstat.txt (911011 bytes)
6332024-11-15T02:08:31.865Zupload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 911011 -> 926558
6342024-11-15T02:08:32.869Zuploaded: /tmp/debug/prstat.txt
6352024-11-15T02:08:32.872Zuploading: /tmp/debug/psrinfo.txt (1528 bytes)
6362024-11-15T02:08:33.877Zuploaded: /tmp/debug/psrinfo.txt
6372024-11-15T02:08:33.880Zuploading: /tmp/debug/upstairs-info.txt (93784 bytes)
6382024-11-15T02:08:34.892Zuploaded: /tmp/debug/upstairs-info.txt