|
|
|
1 | 2024-09-27T17:08:43.889Z | job dependencies complete; ready to run (waiting for 14 m 35 s) |
2 | 2024-09-27T17:09:41.815Z | job assigned to worker 01J8T79TZ98ARZXF9AJ1D7W0D7 (queued for 57 s) |
3 | 2024-09-27T17:09:49.684Z | downloading input: /input/build/work/bins/crucible-downstairs.gz |
4 | 2024-09-27T17:09:50.808Z | downloaded input: /input/build/work/bins/crucible-downstairs.gz |
5 | 2024-09-27T17:09:50.808Z | downloading input: /input/build/work/bins/crucible-hammer.gz |
6 | 2024-09-27T17:09:51.893Z | downloaded input: /input/build/work/bins/crucible-hammer.gz |
7 | 2024-09-27T17:09:51.893Z | downloading input: /input/build/work/bins/crutest.gz |
8 | 2024-09-27T17:09:53.019Z | downloaded input: /input/build/work/bins/crutest.gz |
9 | 2024-09-27T17:09:53.019Z | downloading input: /input/build/work/bins/dsc.gz |
10 | 2024-09-27T17:09:53.722Z | downloaded input: /input/build/work/bins/dsc.gz |
11 | 2024-09-27T17:09:53.725Z | downloading input: /input/build/work/scripts/perf-downstairs-tick.d |
12 | 2024-09-27T17:09:53.740Z | downloaded input: /input/build/work/scripts/perf-downstairs-tick.d |
13 | 2024-09-27T17:09:53.740Z | downloading input: /input/build/work/scripts/test_ds.sh |
14 | 2024-09-27T17:09:53.755Z | downloaded input: /input/build/work/scripts/test_ds.sh |
15 | 2024-09-27T17:09:53.755Z | downloading input: /input/build/work/scripts/test_live_repair.sh |
16 | 2024-09-27T17:09:53.773Z | downloaded input: /input/build/work/scripts/test_live_repair.sh |
17 | 2024-09-27T17:09:53.773Z | downloading input: /input/build/work/scripts/test_repair.sh |
18 | 2024-09-27T17:09:53.791Z | downloaded input: /input/build/work/scripts/test_repair.sh |
19 | 2024-09-27T17:09:53.791Z | downloading input: /input/build/work/scripts/test_replay.sh |
20 | 2024-09-27T17:09:53.810Z | downloaded input: /input/build/work/scripts/test_replay.sh |
21 | 2024-09-27T17:09:53.811Z | downloading input: /input/build/work/scripts/test_up.sh |
22 | 2024-09-27T17:09:53.828Z | downloaded input: /input/build/work/scripts/test_up.sh |
23 | 2024-09-27T17:09:53.829Z | downloading input: /input/build/work/scripts/upstairs_info.d |
24 | 2024-09-27T17:09:53.843Z | downloaded input: /input/build/work/scripts/upstairs_info.d |
25 | 2024-09-27T17:09:53.845Z | downloading input: /input/build/tmp/cargo-test-out.log |
26 | 2024-09-27T17:09:54.612Z | downloaded input: /input/build/tmp/cargo-test-out.log |
|
27 | 2024-09-27T17:09:54.612Z | starting task 0: "setup" |
28 | 2024-09-27T17:09:54.617Z | ++ uname -s |
29 | 2024-09-27T17:09:54.620Z | + kern=SunOS |
30 | 2024-09-27T17:09:54.620Z | + case "$kern" in |
31 | 2024-09-27T17:09:54.620Z | + groupadd -g 12345 build |
32 | 2024-09-27T17:09:54.622Z | + useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build |
33 | 2024-09-27T17:09:56.632Z | + zfs create -o mountpoint=/work rpool/work |
34 | 2024-09-27T17:09:56.904Z | ++ awk '$2 == "/home" { print $3 }' /etc/mnttab |
35 | 2024-09-27T17:09:56.906Z | + home_fs=zfs |
36 | 2024-09-27T17:09:56.906Z | + [[ zfs == autofs ]] |
37 | 2024-09-27T17:09:56.906Z | + mkdir -p /home/build |
38 | 2024-09-27T17:09:56.909Z | + chown build:build /home/build /work |
39 | 2024-09-27T17:09:58.911Z | + chmod 0700 /home/build /work |
40 | 2024-09-27T17:09:58.915Z | process exited: duration 4304 ms, exit code 0 |
|
41 | 2024-09-27T17:09:58.924Z | starting task 1: "authentication" |
42 | 2024-09-27T17:09:58.939Z | process exited: duration 17 ms, exit code 0 |
|
43 | 2024-09-27T17:09:58.945Z | starting task 2: "build" |
44 | 2024-09-27T17:09:58.951Z | + banner cores |
45 | 2024-09-27T17:09:58.969Z | |
46 | 2024-09-27T17:09:58.969Z | #### #### ##### ###### #### |
47 | 2024-09-27T17:09:58.969Z | # # # # # # # # |
48 | 2024-09-27T17:09:58.969Z | # # # # # ##### #### |
49 | 2024-09-27T17:09:58.969Z | # # # ##### # # |
50 | 2024-09-27T17:09:58.969Z | # # # # # # # # # |
51 | 2024-09-27T17:09:58.969Z | #### #### # # ###### #### |
52 | 2024-09-27T17:09:58.969Z | |
53 | 2024-09-27T17:09:58.973Z | + pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid |
54 | 2024-09-27T17:09:58.979Z | + echo 'input bins dir contains:' |
55 | 2024-09-27T17:09:58.982Z | input bins dir contains: |
56 | 2024-09-27T17:09:58.982Z | + ls -ltr /input/build/work/bins |
57 | 2024-09-27T17:09:58.982Z | total 771528 |
58 | 2024-09-27T17:09:58.982Z | -rw-r--r-- 1 root root 110734592 Sep 27 17:09 crucible-downstairs.gz |
59 | 2024-09-27T17:09:58.982Z | -rw-r--r-- 1 root root 105905316 Sep 27 17:09 crucible-hammer.gz |
60 | 2024-09-27T17:09:58.982Z | -rw-r--r-- 1 root root 111383517 Sep 27 17:09 crutest.gz |
61 | 2024-09-27T17:09:58.982Z | -rw-r--r-- 1 root root 66745083 Sep 27 17:09 dsc.gz |
62 | 2024-09-27T17:09:58.983Z | + banner unpack |
63 | 2024-09-27T17:09:58.986Z | |
64 | 2024-09-27T17:09:58.986Z | # # # # ##### ## #### # # |
65 | 2024-09-27T17:09:58.986Z | # # ## # # # # # # # # # |
66 | 2024-09-27T17:09:58.986Z | # # # # # # # # # # #### |
67 | 2024-09-27T17:09:58.986Z | # # # # # ##### ###### # # # |
68 | 2024-09-27T17:09:58.986Z | # # # ## # # # # # # # |
69 | 2024-09-27T17:09:58.986Z | #### # # # # # #### # # |
70 | 2024-09-27T17:09:58.986Z | |
71 | 2024-09-27T17:09:58.986Z | + mkdir -p /var/tmp/bins |
72 | 2024-09-27T17:09:58.986Z | + for t in "$input/bins/"*.gz |
73 | 2024-09-27T17:09:58.989Z | ++ basename /input/build/work/bins/crucible-downstairs.gz |
74 | 2024-09-27T17:09:58.989Z | + b=crucible-downstairs.gz |
75 | 2024-09-27T17:09:58.989Z | + b=crucible-downstairs |
76 | 2024-09-27T17:09:58.989Z | + gunzip |
77 | 2024-09-27T17:10:01.278Z | + chmod +x /var/tmp/bins/crucible-downstairs |
78 | 2024-09-27T17:10:01.281Z | + for t in "$input/bins/"*.gz |
79 | 2024-09-27T17:10:01.285Z | ++ basename /input/build/work/bins/crucible-hammer.gz |
80 | 2024-09-27T17:10:01.285Z | + b=crucible-hammer.gz |
81 | 2024-09-27T17:10:01.285Z | + b=crucible-hammer |
82 | 2024-09-27T17:10:01.285Z | + gunzip |
83 | 2024-09-27T17:10:03.462Z | + chmod +x /var/tmp/bins/crucible-hammer |
84 | 2024-09-27T17:10:03.465Z | + for t in "$input/bins/"*.gz |
85 | 2024-09-27T17:10:03.465Z | ++ basename /input/build/work/bins/crutest.gz |
86 | 2024-09-27T17:10:03.468Z | + b=crutest.gz |
87 | 2024-09-27T17:10:03.468Z | + b=crutest |
88 | 2024-09-27T17:10:03.468Z | + gunzip |
89 | 2024-09-27T17:10:05.854Z | + chmod +x /var/tmp/bins/crutest |
90 | 2024-09-27T17:10:05.857Z | + for t in "$input/bins/"*.gz |
91 | 2024-09-27T17:10:05.860Z | ++ basename /input/build/work/bins/dsc.gz |
92 | 2024-09-27T17:10:05.860Z | + b=dsc.gz |
93 | 2024-09-27T17:10:05.860Z | + b=dsc |
94 | 2024-09-27T17:10:05.860Z | + gunzip |
95 | 2024-09-27T17:10:07.230Z | + chmod +x /var/tmp/bins/dsc |
96 | 2024-09-27T17:10:07.234Z | + export BINDIR=/var/tmp/bins |
97 | 2024-09-27T17:10:07.234Z | + BINDIR=/var/tmp/bins |
98 | 2024-09-27T17:10:07.234Z | + jobpid=1137 |
99 | 2024-09-27T17:10:07.234Z | + echo 'Setup debug logging' |
100 | 2024-09-27T17:10:07.234Z | Setup debug logging |
101 | 2024-09-27T17:10:07.234Z | + mkdir /tmp/debug |
102 | 2024-09-27T17:10:07.234Z | + sleep 7200 |
103 | 2024-09-27T17:10:07.239Z | + psrinfo -v |
104 | 2024-09-27T17:10:07.242Z | + df -h |
105 | 2024-09-27T17:10:07.276Z | + prstat -d d -mLc 1 |
106 | 2024-09-27T17:10:07.281Z | + iostat -T d -xn 1 |
107 | 2024-09-27T17:10:07.281Z | + mpstat -T d 1 |
108 | 2024-09-27T17:10:07.281Z | + vmstat -T d -p 1 |
109 | 2024-09-27T17:10:07.281Z | + pfexec dtrace -Z -s /input/build/work/scripts/perf-downstairs-tick.d |
110 | 2024-09-27T17:10:07.281Z | + banner test_up_encrypted |
111 | 2024-09-27T17:10:07.281Z | + pfexec dtrace -Z -s /input/build/work/scripts/upstairs_info.d |
112 | 2024-09-27T17:10:07.285Z | |
113 | 2024-09-27T17:10:07.285Z | ##### ###### #### ##### # # ##### ###### # # |
114 | 2024-09-27T17:10:07.285Z | # # # # # # # # # ## # |
115 | 2024-09-27T17:10:07.285Z | # ##### #### # # # # # ##### # # # |
116 | 2024-09-27T17:10:07.285Z | # # # # # # ##### # # # # |
117 | 2024-09-27T17:10:07.285Z | # # # # # # # # # # ## |
118 | 2024-09-27T17:10:07.285Z | # ###### #### # ####### #### # ####### ###### # # |
119 | 2024-09-27T17:10:07.285Z | |
120 | 2024-09-27T17:10:07.285Z | + ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted |
121 | 2024-09-27T17:10:07.292Z | /input/build/work |
122 | 2024-09-27T17:10:07.321Z | Turn off color for downstairs dump |
123 | 2024-09-27T17:10:07.347Z | Upstairs using key: rWzZG8E/8XgqIm3XrvGn0g0nYOJ/sNiEzEskSoS4E9U= |
124 | 2024-09-27T17:10:07.350Z | dsc output goes to /tmp/test_up-build/dsc-out.txt |
125 | 2024-09-27T17:10:07.350Z | Creating three downstairs regions |
126 | 2024-09-27T17:10:07.731Z | Starting three downstairs |
127 | 2024-09-27T17:10:12.739Z | Disable automatic restart on all downstairs |
128 | 2024-09-27T17:10:12.799Z | |
129 | 2024-09-27T17:10:12.799Z | Begin tests, output goes to /tmp/test_up-build/test_up_out.txt |
130 | 2024-09-27T17:10:12.804Z | Running test: span |
131 | 2024-09-27T17:10:16.987Z | Completed test: span |
132 | 2024-09-27T17:10:21.994Z | Running test: big |
133 | 2024-09-27T17:10:26.424Z | Completed test: big |
134 | 2024-09-27T17:10:31.410Z | Running test: dep |
135 | 2024-09-27T17:11:25.664Z | Completed test: dep |
136 | 2024-09-27T17:11:30.671Z | Running test: balloon |
137 | 2024-09-27T17:12:17.096Z | Completed test: balloon |
138 | 2024-09-27T17:12:22.102Z | Running test: deactivate |
139 | 2024-09-27T17:13:12.853Z | Completed test: deactivate |
140 | 2024-09-27T17:13:17.860Z | Running hammer |
141 | 2024-09-27T17:13:31.603Z | Run repair tests |
142 | 2024-09-27T17:13:31.607Z | /var/tmp/bins/crutest fill -g 31 -q --key rWzZG8E/8XgqIm3XrvGn0g0nYOJ/sNiEzEskSoS4E9U= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up-build/verify_file |
143 | 2024-09-27T17:13:31.623Z | {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.097518319Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"task":"crutest"} |
144 | 2024-09-27T17:13:31.666Z | {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008810, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.14084703Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"task":"crutest"} |
145 | 2024-09-27T17:13:31.669Z | {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.14149239Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217} |
146 | 2024-09-27T17:13:31.669Z | {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.141549587Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217} |
147 | 2024-09-27T17:13:31.669Z | {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.141580466Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217} |
148 | 2024-09-27T17:13:31.669Z | {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.141599825Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217} |
149 | 2024-09-27T17:13:31.669Z | {"msg":"Crucible 2e5b70dc-6477-4a01-8338-42160bb5f582 has session id: de096f19-2c59-4e9a-b1d5-aec09578d4a9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.142024254Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
150 | 2024-09-27T17:13:31.669Z | {"msg":"Upstairs opts: Upstairs UUID: 2e5b70dc-6477-4a01-8338-42160bb5f582, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.142079082Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
151 | 2024-09-27T17:13:31.669Z | {"msg":"Crucible stats registered with UUID: 2e5b70dc-6477-4a01-8338-42160bb5f582","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.142097571Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
152 | 2024-09-27T17:13:31.699Z | {"msg":"The guest has requested activation with gen:31","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.173507159Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217} |
153 | 2024-09-27T17:13:31.699Z | {"msg":"2e5b70dc-6477-4a01-8338-42160bb5f582 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.173568586Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
154 | 2024-09-27T17:13:31.699Z | {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.173591075Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"0","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
155 | 2024-09-27T17:13:31.699Z | {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.173620733Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"0","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
156 | 2024-09-27T17:13:31.701Z | {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.173640362Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"1","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
157 | 2024-09-27T17:13:31.702Z | {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.17368Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"1","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
158 | 2024-09-27T17:13:31.702Z | {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.173712099Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"2","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
159 | 2024-09-27T17:13:31.702Z | {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.173760737Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"2","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
160 | 2024-09-27T17:13:31.702Z | {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.173791405Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
161 | 2024-09-27T17:13:31.702Z | {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.173831643Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
162 | 2024-09-27T17:13:31.702Z | {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.173863552Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
163 | 2024-09-27T17:13:31.702Z | {"msg":"ds_connection connected from Ok(127.0.0.1:34209)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.174182757Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
164 | 2024-09-27T17:13:31.702Z | {"msg":"ds_connection connected from Ok(127.0.0.1:51283)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.174206275Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
165 | 2024-09-27T17:13:31.702Z | {"msg":"ds_connection connected from Ok(127.0.0.1:37184)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.174231404Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
166 | 2024-09-27T17:13:31.702Z | {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.174862194Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"1","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
167 | 2024-09-27T17:13:31.702Z | {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.174900052Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"1","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
168 | 2024-09-27T17:13:31.702Z | {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.174915522Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"2","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
169 | 2024-09-27T17:13:31.702Z | {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.174929571Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"2","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
170 | 2024-09-27T17:13:31.702Z | {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.174998478Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"0","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
171 | 2024-09-27T17:13:31.702Z | {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.175029346Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"0","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
172 | 2024-09-27T17:13:31.702Z | {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.175599539Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"1","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
173 | 2024-09-27T17:13:31.702Z | {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.175670936Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"0","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
174 | 2024-09-27T17:13:31.702Z | {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.175707674Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"2","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
175 | 2024-09-27T17:13:31.702Z | {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176097755Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"0","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
176 | 2024-09-27T17:13:31.702Z | {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176135634Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"0","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
177 | 2024-09-27T17:13:31.702Z | {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176151223Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
178 | 2024-09-27T17:13:31.702Z | {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176183522Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"2","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
179 | 2024-09-27T17:13:31.702Z | {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176200951Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"2","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
180 | 2024-09-27T17:13:31.702Z | {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176231979Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
181 | 2024-09-27T17:13:31.702Z | {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176249388Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"1","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
182 | 2024-09-27T17:13:31.702Z | {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176279207Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"1","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
183 | 2024-09-27T17:13:31.702Z | {"msg":"[0]R flush_numbers: [1688, 1688, 1688, 1688, 1688]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176299896Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
184 | 2024-09-27T17:13:31.702Z | {"msg":"[0]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176329544Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
185 | 2024-09-27T17:13:31.702Z | {"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176344604Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
186 | 2024-09-27T17:13:31.702Z | {"msg":"[1]R flush_numbers: [1688, 1688, 1688, 1688, 1688]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176357993Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
187 | 2024-09-27T17:13:31.702Z | {"msg":"Wait for a query_work_queue command to finish before sending IO |
188 | 2024-09-27T17:13:31.705Z | [1]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176385582Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
189 | 2024-09-27T17:13:31.705Z | {"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.17642531Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
190 | 2024-09-27T17:13:31.705Z | {"msg":"[2]R flush_numbers: [1688, 1688, 1688, 1688, 1688]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176449179Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
191 | 2024-09-27T17:13:31.705Z | {"msg":"[2]R generation: [25, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176477548Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
192 | 2024-09-27T17:13:31.705Z | {"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176492417Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
193 | 2024-09-27T17:13:31.705Z | {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176523285Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
194 | 2024-09-27T17:13:31.705Z | {"msg":"Generation requested: 31 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176542504Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
195 | 2024-09-27T17:13:31.705Z | {"msg":"Next flush: 1689","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176565123Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
196 | 2024-09-27T17:13:31.705Z | {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176588802Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"":"downstairs","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
197 | 2024-09-27T17:13:31.705Z | {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800 |
198 | 2024-09-27T17:13:31.705Z | ,"time":"2024-09-27T17:13:30.176614241Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
199 | 2024-09-27T17:13:31.705Z | {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.17662741Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"0","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
200 | 2024-09-27T17:13:31.705Z | {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.17664082Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"1","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
201 | 2024-09-27T17:13:31.706Z | Fill test |
202 | 2024-09-27T17:13:31.706Z | {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176663959Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"client":"2","":"downstairs","session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
203 | 2024-09-27T17:13:31.706Z | {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176678268Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
204 | 2024-09-27T17:13:31.706Z | {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176697447Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
205 | 2024-09-27T17:13:31.706Z | {"msg":"2e5b70dc-6477-4a01-8338-42160bb5f582 is now active with session: de096f19-2c59-4e9a-b1d5-aec09578d4a9","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176710616Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
206 | 2024-09-27T17:13:31.706Z | {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176729676Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217,"session_id":"de096f19-2c59-4e9a-b1d5-aec09578d4a9"} |
207 | 2024-09-27T17:13:31.706Z | {"msg":"The guest has finished waiting for activation with:31","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:30.176742245Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217} |
208 | 2024-09-27T17:13:31.782Z | Read and Verify all blocks (0..50 range:false) |
209 | 2024-09-27T17:13:31.824Z | Wrote out file "/var/tmp/test_up-build/verify_file" |
210 | 2024-09-27T17:13:31.824Z | CLIENT: Tests done. All submitted work has been ACK'd |
211 | 2024-09-27T17:13:31.826Z | ---------------------------------------------------------------- |
212 | 2024-09-27T17:13:31.826Z | Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:1 |
213 | 2024-09-27T17:13:31.827Z | States: Active Active Active |
214 | 2024-09-27T17:13:31.827Z | GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY |
215 | 2024-09-27T17:13:31.827Z | 3 Acked 1002 Read 50 Done Done Done false |
216 | 2024-09-27T17:13:31.827Z | STATES DS:0 DS:1 DS:2 TOTAL |
217 | 2024-09-27T17:13:31.827Z | Sent 0 0 0 0 |
218 | 2024-09-27T17:13:31.827Z | Done 1 1 1 3 |
219 | 2024-09-27T17:13:31.827Z | Skipped 0 0 0 0 |
220 | 2024-09-27T17:13:31.827Z | Error 0 0 0 0 |
221 | 2024-09-27T17:13:31.827Z | Last Flush: 1001 1001 1001 |
222 | 2024-09-27T17:13:31.827Z | Downstairs last five completed: 1001 1000 |
223 | 2024-09-27T17:13:31.827Z | Upstairs last five completed: 3 2 1 |
224 | 2024-09-27T17:13:31.827Z | CLIENT: Up:0 ds:1 act:3 |
225 | 2024-09-27T17:13:35.825Z | ---------------------------------------------------------------- |
226 | 2024-09-27T17:13:35.825Z | Crucible gen:31 GIO:true work queues: Upstairs:0 downstairs:0 |
227 | 2024-09-27T17:13:35.825Z | Downstairs last five completed: 1003 1002 1001 1000 |
228 | 2024-09-27T17:13:35.825Z | Upstairs last five completed: 4 3 2 1 |
229 | 2024-09-27T17:13:35.825Z | CLIENT: Up:0 ds:0 act:3 |
230 | 2024-09-27T17:13:35.825Z | CLIENT: All crucible jobs finished, exiting program |
231 | 2024-09-27T17:13:35.828Z | {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:34.299117434Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1217} |
232 | 2024-09-27T17:13:35.835Z | Repair setup passed |
233 | 2024-09-27T17:13:35.839Z | Copy the 8830 file |
234 | 2024-09-27T17:13:35.842Z | cp -r /var/tmp/test_up-build/8830 /var/tmp/test_up-build/previous |
235 | 2024-09-27T17:13:35.844Z | /var/tmp/bins/crutest repair -g 32 -q --key rWzZG8E/8XgqIm3XrvGn0g0nYOJ/sNiEzEskSoS4E9U= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file |
236 | 2024-09-27T17:13:35.860Z | {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.332754656Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"task":"crutest"} |
237 | 2024-09-27T17:13:35.902Z | {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008810, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.376188832Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"task":"crutest"} |
238 | 2024-09-27T17:13:35.906Z | {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.376767565Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223} |
239 | 2024-09-27T17:13:35.906Z | {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.376821942Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223} |
240 | 2024-09-27T17:13:35.906Z | {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.37685997Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223} |
241 | 2024-09-27T17:13:35.906Z | {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.376905298Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223} |
242 | 2024-09-27T17:13:35.906Z | {"msg":"Crucible 1009559a-3d73-4efc-99dc-aa83b2b3266b has session id: 81b154a5-6708-4c9c-b0d3-c5edfff6162c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.377315679Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
243 | 2024-09-27T17:13:35.906Z | {"msg":"Upstairs opts: Upstairs UUID: 1009559a-3d73-4efc-99dc-aa83b2b3266b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.377360466Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
244 | 2024-09-27T17:13:35.906Z | {"msg":"Crucible stats registered with UUID: 1009559a-3d73-4efc-99dc-aa83b2b3266b","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.377393375Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
245 | 2024-09-27T17:13:35.935Z | {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.408419221Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223} |
246 | 2024-09-27T17:13:35.935Z | {"msg":"1009559a-3d73-4efc-99dc-aa83b2b3266b active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.408461049Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
247 | 2024-09-27T17:13:35.935Z | {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.408479298Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"0","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
248 | 2024-09-27T17:13:35.940Z | {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.408512696Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"0","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
249 | 2024-09-27T17:13:35.941Z | {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.408553015Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"1","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
250 | 2024-09-27T17:13:35.941Z | {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.408584653Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"1","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
251 | 2024-09-27T17:13:35.941Z | {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.408617141Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"2","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
252 | 2024-09-27T17:13:35.941Z | {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.40865158Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"2","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
253 | 2024-09-27T17:13:35.941Z | {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.408694228Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
254 | 2024-09-27T17:13:35.941Z | {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.408725716Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
255 | 2024-09-27T17:13:35.941Z | {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.408977794Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
256 | 2024-09-27T17:13:35.942Z | {"msg":"ds_connection connected from Ok(127.0.0.1:61343)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.409026942Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
257 | 2024-09-27T17:13:35.942Z | {"msg":"ds_connection connected from Ok(127.0.0.1:41168)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.40906894Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
258 | 2024-09-27T17:13:35.942Z | {"msg":"ds_connection connected from Ok(127.0.0.1:45558)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.409093489Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
259 | 2024-09-27T17:13:35.942Z | {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.409618004Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"1","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
260 | 2024-09-27T17:13:35.942Z | {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.409655422Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"1","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
261 | 2024-09-27T17:13:35.942Z | {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.409670731Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"2","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
262 | 2024-09-27T17:13:35.942Z | {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.4097035Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"2","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
263 | 2024-09-27T17:13:35.942Z | {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.409749618Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"0","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
264 | 2024-09-27T17:13:35.942Z | {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.409783686Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"0","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
265 | 2024-09-27T17:13:35.943Z | {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410236224Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"0","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
266 | 2024-09-27T17:13:35.943Z | {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410358469Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"1","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
267 | 2024-09-27T17:13:35.943Z | {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410568179Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"2","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
268 | 2024-09-27T17:13:35.943Z | {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410774309Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"1","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
269 | 2024-09-27T17:13:35.943Z | {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410827456Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"1","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
270 | 2024-09-27T17:13:35.943Z | {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410865075Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
271 | 2024-09-27T17:13:35.944Z | {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410885974Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"0","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
272 | 2024-09-27T17:13:35.944Z | {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410899853Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"0","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
273 | 2024-09-27T17:13:35.944Z | {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410926932Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
274 | 2024-09-27T17:13:35.944Z | {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410943261Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"2","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
275 | 2024-09-27T17:13:35.944Z | {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.41096735Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"2","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
276 | 2024-09-27T17:13:35.944Z | {"msg":"[0]R flush_numbers: [1689, 1689, 1689, 1689, 1689]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.410981329Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
277 | 2024-09-27T17:13:35.944Z | {"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411001588Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
278 | 2024-09-27T17:13:35.944Z | {"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411015407Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
279 | 2024-09-27T17:13:35.945Z | {"msg":"[1]R flush_numbers: [1689, 1689, 1689, 1689, 1689]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411029357Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
280 | 2024-09-27T17:13:35.945Z | {"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411042596Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
281 | 2024-09-27T17:13:35.945Z | {"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411062435Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
282 | 2024-09-27T17:13:35.945Z | {"msg":"[2]R flush_numbers: [1689, 1689, 1689, 1689, 1689]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411086634Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
283 | 2024-09-27T17:13:35.945Z | {"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411111333Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
284 | 2024-09-27T17:13:35.945Z | {"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411133862Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
285 | 2024-09-27T17:13:35.945Z | {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411154181Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
286 | 2024-09-27T17:13:35.945Z | {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.41116775Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
287 | 2024-09-27T17:13:35.945Z | Wait for a query_work_queue command to finish before sending IO |
288 | 2024-09-27T17:13:35.945Z | {"msg":"Next flush: 1690","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411201389Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
289 | 2024-09-27T17:13:35.945Z | {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411221148Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"":"downstairs","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
290 | 2024-09-27T17:13:35.946Z | {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411234587Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
291 | 2024-09-27T17:13:35.946Z | {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411253546Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"0","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
292 | 2024-09-27T17:13:35.946Z | {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411267786Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"1","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
293 | 2024-09-27T17:13:35.946Z | {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411288134Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"client":"2","":"downstairs","session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
294 | 2024-09-27T17:13:35.946Z | {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411301474Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
295 | 2024-09-27T17:13:35.946Z | {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411320633Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
296 | 2024-09-27T17:13:35.946Z | {"msg":"1009559a-3d73-4efc-99dc-aa83b2b3266b is now active with session: 81b154a5-6708-4c9c-b0d3-c5edfff6162c","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411333572Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
297 | 2024-09-27T17:13:35.946Z | {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411345452Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223,"session_id":"81b154a5-6708-4c9c-b0d3-c5edfff6162c"} |
298 | 2024-09-27T17:13:35.946Z | {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.411357451Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223} |
299 | 2024-09-27T17:13:35.946Z | Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800 |
300 | 2024-09-27T17:13:35.946Z | Loading write count information from file "/var/tmp/test_up-build/verify_file" |
301 | 2024-09-27T17:13:35.947Z | Run Repair workload |
302 | 2024-09-27T17:13:35.947Z | 01/10 Write block 6 len 40960 data: 2 2 2 2 2 2 2 2 2 2 |
303 | 2024-09-27T17:13:35.947Z | 02/10 Read block 45 len 16384 |
304 | 2024-09-27T17:13:35.954Z | 03/10 Read block 4 len 24576 |
305 | 2024-09-27T17:13:35.958Z | 04/10 Write block 41 len 28672 data: 2 2 2 2 2 2 2 |
306 | 2024-09-27T17:13:35.958Z | 05/10 Write block 10 len 4096 data: 3 |
307 | 2024-09-27T17:13:35.958Z | 06/10 Write block 16 len 32768 data: 2 2 2 2 2 2 2 2 |
308 | 2024-09-27T17:13:35.958Z | 07/10 Write block 43 len 16384 data: 3 3 3 3 |
309 | 2024-09-27T17:13:35.958Z | 08/10 Read block 15 len 40960 |
310 | 2024-09-27T17:13:35.973Z | 09/10 Write block 15 len 32768 data: 3 3 3 3 3 3 3 3 |
311 | 2024-09-27T17:13:35.976Z | 10/10 Read block 19 len 36864 |
312 | 2024-09-27T17:13:35.987Z | ---------------------------------------------------------------- |
313 | 2024-09-27T17:13:35.987Z | Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10 |
314 | 2024-09-27T17:13:35.987Z | States: Active Active Active |
315 | 2024-09-27T17:13:35.987Z | GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY |
316 | 2024-09-27T17:13:35.987Z | 1 Acked 1000 Write 10 Done Done Done false |
317 | 2024-09-27T17:13:35.987Z | 2 Acked 1001 Read 4 Done Done Done false |
318 | 2024-09-27T17:13:35.987Z | 3 Acked 1002 Read 6 Done Done Done false |
319 | 2024-09-27T17:13:35.988Z | 4 Acked 1003 Write 7 Done Done Done false |
320 | 2024-09-27T17:13:35.988Z | 5 Acked 1004 Write 1 Done Done Done false |
321 | 2024-09-27T17:13:35.988Z | 6 Acked 1005 Write 8 Done Done Done false |
322 | 2024-09-27T17:13:35.988Z | 7 Acked 1006 Write 4 Done Done Done false |
323 | 2024-09-27T17:13:35.988Z | 8 Acked 1007 Read 10 Done Done Done false |
324 | 2024-09-27T17:13:35.988Z | 9 Acked 1008 Write 8 Done Done Done false |
325 | 2024-09-27T17:13:35.988Z | 10 Acked 1009 Read 9 Done Sent Sent false |
326 | 2024-09-27T17:13:35.988Z | STATES DS:0 DS:1 DS:2 TOTAL |
327 | 2024-09-27T17:13:35.988Z | Sent 0 1 1 2 |
328 | 2024-09-27T17:13:35.988Z | Done 10 9 9 28 |
329 | 2024-09-27T17:13:35.988Z | Skipped 0 0 0 0 |
330 | 2024-09-27T17:13:35.988Z | Error 0 0 0 0 |
331 | 2024-09-27T17:13:35.988Z | Last Flush: 0 0 0 |
332 | 2024-09-27T17:13:35.988Z | Downstairs last five completed: |
333 | 2024-09-27T17:13:35.988Z | Upstairs last five completed: 10 9 8 7 6 |
334 | 2024-09-27T17:13:35.991Z | {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:34.46146669Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1223} |
335 | 2024-09-27T17:13:35.991Z | Wrote out file "/var/tmp/test_up-build/verify_file" |
336 | 2024-09-27T17:13:35.994Z | Repair part 1 passed |
337 | 2024-09-27T17:13:35.994Z | |
338 | 2024-09-27T17:13:35.994Z | Kill the current downstairs |
339 | 2024-09-27T17:13:36.103Z | rm -rf /var/tmp/test_up-build/8830 |
340 | 2024-09-27T17:13:36.104Z | Now put back the original so we have a mismatch |
341 | 2024-09-27T17:13:36.104Z | mv /var/tmp/test_up-build/previous /var/tmp/test_up-build/8830 |
342 | 2024-09-27T17:13:36.107Z | Restart downstairs with old directory |
343 | 2024-09-27T17:13:36.163Z | /var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 |
344 | 2024-09-27T17:13:36.180Z | EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF |
345 | 2024-09-27T17:13:36.182Z | 0 000-009 31 31 31 1689 1689 1689 T T F <--- |
346 | 2024-09-27T17:13:36.182Z | 1 010-019 31 31 31 1689 1689 1689 T T F <--- |
347 | 2024-09-27T17:13:36.182Z | 2 020-029 31 31 31 1689 1689 1689 T T F <--- |
348 | 2024-09-27T17:13:36.183Z | 3 030-039 31 31 31 1689 1689 1689 F F F |
349 | 2024-09-27T17:13:36.183Z | 4 040-049 31 31 31 1689 1689 1689 T T F <--- |
350 | 2024-09-27T17:13:36.183Z | Max gen: 31, Max flush: 1689 |
351 | 2024-09-27T17:13:36.183Z | Error: Difference in extent metadata found! |
352 | 2024-09-27T17:13:36.186Z | dump test found error as expected |
353 | 2024-09-27T17:13:36.186Z | |
354 | 2024-09-27T17:13:36.186Z | |
355 | 2024-09-27T17:13:36.186Z | /var/tmp/bins/crutest verify --range -g 33 -q --key rWzZG8E/8XgqIm3XrvGn0g0nYOJ/sNiEzEskSoS4E9U= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up-build/verify_file --verify-in /var/tmp/test_up-build/verify_file |
356 | 2024-09-27T17:13:36.198Z | {"msg":"look at: http://127.0.0.1:12810 ","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.672050115Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"task":"crutest"} |
357 | 2024-09-27T17:13:36.241Z | {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008810, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.71551621Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"task":"crutest"} |
358 | 2024-09-27T17:13:36.243Z | {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.716288554Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230} |
359 | 2024-09-27T17:13:36.243Z | {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b12f70ee7c1993c73ac4256d99ff80ccbe4bac\",\n git_commit_timestamp: \"2024-09-27T16:52:31.000000000Z\",\n git_branch: \"update-omicron-common\",\n rustc_semver: \"1.80.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"051478957371ee0084a7c0913941d2a8c4757bb9\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.716330712Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230} |
360 | 2024-09-27T17:13:36.244Z | {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.716347831Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230} |
361 | 2024-09-27T17:13:36.244Z | {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.7163704Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230} |
362 | 2024-09-27T17:13:36.244Z | {"msg":"Crucible c7d40530-c950-4128-a645-c384a86d62f2 has session id: 7e4dc9d6-72a4-443b-80a5-97e066c91131","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.716822138Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
363 | 2024-09-27T17:13:36.244Z | {"msg":"Upstairs opts: Upstairs UUID: c7d40530-c950-4128-a645-c384a86d62f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.716885155Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
364 | 2024-09-27T17:13:36.244Z | {"msg":"Crucible stats registered with UUID: c7d40530-c950-4128-a645-c384a86d62f2","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.716905864Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
365 | 2024-09-27T17:13:36.278Z | {"msg":"The guest has requested activation with gen:33","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748034875Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230} |
366 | 2024-09-27T17:13:36.278Z | {"msg":"c7d40530-c950-4128-a645-c384a86d62f2 active request set","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748077563Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
367 | 2024-09-27T17:13:36.278Z | {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748096272Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
368 | 2024-09-27T17:13:36.281Z | {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.74813942Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
369 | 2024-09-27T17:13:36.282Z | {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748186878Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
370 | 2024-09-27T17:13:36.282Z | {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748223296Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
371 | 2024-09-27T17:13:36.282Z | {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748315272Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
372 | 2024-09-27T17:13:36.282Z | {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748380379Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
373 | 2024-09-27T17:13:36.282Z | {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748431907Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
374 | 2024-09-27T17:13:36.282Z | {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748525632Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
375 | 2024-09-27T17:13:36.282Z | {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748615808Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
376 | 2024-09-27T17:13:36.282Z | {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:34.748685844Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
377 | 2024-09-27T17:13:36.282Z | {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:34.748746961Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
378 | 2024-09-27T17:13:36.282Z | {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:34.748796559Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
379 | 2024-09-27T17:13:36.282Z | {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748817618Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
380 | 2024-09-27T17:13:36.282Z | {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748843737Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
381 | 2024-09-27T17:13:36.282Z | {"msg":"ds_connection connected from Ok(127.0.0.1:50479)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748862126Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
382 | 2024-09-27T17:13:36.282Z | {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.748933363Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
383 | 2024-09-27T17:13:36.282Z | {"msg":"ds_connection connected from Ok(127.0.0.1:35432)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.74898549Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
384 | 2024-09-27T17:13:36.282Z | {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.74919486Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
385 | 2024-09-27T17:13:36.282Z | {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.749225589Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
386 | 2024-09-27T17:13:36.282Z | {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.749241378Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
387 | 2024-09-27T17:13:36.282Z | {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.749299355Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
388 | 2024-09-27T17:13:36.282Z | {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.749785052Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
389 | 2024-09-27T17:13:36.282Z | {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.749908036Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
390 | 2024-09-27T17:13:36.282Z | {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.750127966Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
391 | 2024-09-27T17:13:36.282Z | {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.750180533Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
392 | 2024-09-27T17:13:36.282Z | {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.750216472Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
393 | 2024-09-27T17:13:36.282Z | {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.75025803Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
394 | 2024-09-27T17:13:36.282Z | {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.750286908Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
395 | 2024-09-27T17:13:36.282Z | {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:34.750317487Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
396 | 2024-09-27T17:13:46.278Z | {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.750432883Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
397 | 2024-09-27T17:13:46.282Z | {"msg":"ds_connection connected from Ok(127.0.0.1:61645)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.750798216Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
398 | 2024-09-27T17:13:46.283Z | {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.75175839Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
399 | 2024-09-27T17:13:46.283Z | {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.751806738Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
400 | 2024-09-27T17:13:46.283Z | {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752346392Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
401 | 2024-09-27T17:13:46.283Z | {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752725774Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
402 | 2024-09-27T17:13:46.283Z | {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752760293Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
403 | 2024-09-27T17:13:46.283Z | {"msg":"[0]R flush_numbers: [1689, 1689, 1689, 1689, 1689]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752780032Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
404 | 2024-09-27T17:13:46.283Z | {"msg":"[0]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752806241Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
405 | 2024-09-27T17:13:46.283Z | {"msg":"[0]R dirty: [true, true, true, false, true]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.75282349Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
406 | 2024-09-27T17:13:46.283Z | {"msg":"[1]R flush_numbers: [1689, 1689, 1689, 1689, 1689]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752841199Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
407 | 2024-09-27T17:13:46.283Z | {"msg":"[1]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752858288Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
408 | 2024-09-27T17:13:46.283Z | {"msg":"[1]R dirty: [true, true, true, false, true]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752883367Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
409 | 2024-09-27T17:13:46.283Z | {"msg":"[2]R flush_numbers: [1689, 1689, 1689, 1689, 1689]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752900416Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
410 | 2024-09-27T17:13:46.283Z | {"msg":"[2]R generation: [31, 31, 31, 31, 31]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752924885Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
411 | 2024-09-27T17:13:46.283Z | {"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752942594Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
412 | 2024-09-27T17:13:46.283Z | {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752959073Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
413 | 2024-09-27T17:13:46.283Z | {"msg":"Generation requested: 33 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.752983862Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
414 | 2024-09-27T17:13:46.283Z | {"msg":"Next flush: 1690","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753000701Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
415 | 2024-09-27T17:13:46.283Z | {"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.75302565Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
416 | 2024-09-27T17:13:46.283Z | {"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753043369Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
417 | 2024-09-27T17:13:46.283Z | {"msg":"extent:0 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753069268Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
418 | 2024-09-27T17:13:46.283Z | {"msg":"extent:0 flush: 1689 1689 1689 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753088957Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
419 | 2024-09-27T17:13:46.283Z | {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753107836Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
420 | 2024-09-27T17:13:46.283Z | {"msg":"extent:0 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753133705Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
421 | 2024-09-27T17:13:46.283Z | {"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753153044Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
422 | 2024-09-27T17:13:46.283Z | {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753178943Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
423 | 2024-09-27T17:13:46.283Z | {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753197662Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
424 | 2024-09-27T17:13:46.283Z | {"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.75323171Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
425 | 2024-09-27T17:13:46.283Z | {"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753258879Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
426 | 2024-09-27T17:13:46.284Z | {"msg":"extent:1 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753277028Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
427 | 2024-09-27T17:13:46.284Z | {"msg":"extent:1 flush: 1689 1689 1689 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753303667Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
428 | 2024-09-27T17:13:46.284Z | {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753322846Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
429 | 2024-09-27T17:13:46.284Z | {"msg":"extent:1 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753353704Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
430 | 2024-09-27T17:13:46.284Z | {"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753390833Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
431 | 2024-09-27T17:13:46.284Z | {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753436461Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
432 | 2024-09-27T17:13:46.284Z | {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753470849Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
433 | 2024-09-27T17:13:46.284Z | {"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753504587Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
434 | 2024-09-27T17:13:46.284Z | {"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753540306Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
435 | 2024-09-27T17:13:46.284Z | {"msg":"extent:2 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753572524Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
436 | 2024-09-27T17:13:46.284Z | {"msg":"extent:2 flush: 1689 1689 1689 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753608792Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
437 | 2024-09-27T17:13:46.284Z | {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753642631Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
438 | 2024-09-27T17:13:46.284Z | {"msg":"extent:2 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753708778Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
439 | 2024-09-27T17:13:46.284Z | {"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753743436Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
440 | 2024-09-27T17:13:46.284Z | {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753778714Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
441 | 2024-09-27T17:13:46.284Z | {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753806113Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
442 | 2024-09-27T17:13:46.284Z | {"msg":"Extents 4 dirty","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753822332Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
443 | 2024-09-27T17:13:46.284Z | {"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753842591Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
444 | 2024-09-27T17:13:46.284Z | {"msg":"extent:4 gens: 31 31 31","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753856921Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
445 | 2024-09-27T17:13:46.284Z | {"msg":"extent:4 flush: 1689 1689 1689 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.75387829Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
446 | 2024-09-27T17:13:46.284Z | {"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753893129Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
447 | 2024-09-27T17:13:46.284Z | {"msg":"extent:4 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753913868Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
448 | 2024-09-27T17:13:46.284Z | {"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753928387Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
449 | 2024-09-27T17:13:46.284Z | {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753949226Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
450 | 2024-09-27T17:13:46.284Z | {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753963626Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"mrl":"dirty","":"mend","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
451 | 2024-09-27T17:13:46.284Z | {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753984574Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
452 | 2024-09-27T17:13:46.285Z | {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.753998224Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
453 | 2024-09-27T17:13:46.285Z | {"msg":"Transition from WaitQuorum to Reconcile","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.754011423Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
454 | 2024-09-27T17:13:46.285Z | {"msg":"starting reconciliation 555c472f-4e08-494f-9fc2-8ab755cde469: found 4 extents that need repair","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.754031812Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
455 | 2024-09-27T17:13:46.285Z | {"msg":"Full repair list: {ExtentId(4): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(2): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(0): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }, ExtentId(1): ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.754049942Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
456 | 2024-09-27T17:13:46.288Z | {"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: ExtentId(4), client_id: ClientId(0), flush_number: 1690, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: ExtentId(4), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: ExtentId(4) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: ExtentId(2), client_id: ClientId(0), flush_number: 1690, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: ExtentId(2), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: ExtentId(2) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: ExtentId(0), client_id: ClientId(0), flush_number: 1690, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: ExtentId(0), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: ExtentId(0) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: ExtentId(1), client_id: ClientId(0), flush_number: 1690, gen_number: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: ExtentId(1), source_client_id: ClientId(0), source_repair_address: 0.0.0.0:12810, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: ExtentId(1) }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.754141067Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
457 | 2024-09-27T17:13:46.288Z | {"msg":"no Nexus client from DNS, aborting start notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.754168056Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
458 | 2024-09-27T17:13:46.288Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.754184755Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
459 | 2024-09-27T17:13:46.288Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.754771117Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
460 | 2024-09-27T17:13:46.288Z | {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.755107041Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
461 | 2024-09-27T17:13:46.288Z | {"msg":"sending reconcile request ReconciliationId(2)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.755153649Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
462 | 2024-09-27T17:13:46.288Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.755185808Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
463 | 2024-09-27T17:13:46.294Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.768076395Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
464 | 2024-09-27T17:13:46.297Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.768451987Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
465 | 2024-09-27T17:13:46.297Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.769603192Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
466 | 2024-09-27T17:13:46.297Z | {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.769891009Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
467 | 2024-09-27T17:13:46.297Z | {"msg":"sending reconcile request ReconciliationId(6)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.769939127Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
468 | 2024-09-27T17:13:46.297Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.769978945Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
469 | 2024-09-27T17:13:46.306Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.779848665Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
470 | 2024-09-27T17:13:46.309Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.780253856Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
471 | 2024-09-27T17:13:46.309Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.78122702Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
472 | 2024-09-27T17:13:46.309Z | {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.781528596Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
473 | 2024-09-27T17:13:46.309Z | {"msg":"sending reconcile request ReconciliationId(10)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.781565754Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
474 | 2024-09-27T17:13:46.309Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.781603962Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
475 | 2024-09-27T17:13:46.318Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.791733651Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
476 | 2024-09-27T17:13:46.321Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.792158091Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
477 | 2024-09-27T17:13:46.321Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.793316476Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
478 | 2024-09-27T17:13:46.321Z | {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.793623401Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
479 | 2024-09-27T17:13:46.321Z | {"msg":"sending reconcile request ReconciliationId(14)","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.793674869Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
480 | 2024-09-27T17:13:46.321Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.793706237Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
481 | 2024-09-27T17:13:46.330Z | {"msg":"no Nexus client from DNS, aborting progress notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.803726221Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
482 | 2024-09-27T17:13:46.333Z | {"msg":"done with reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.804046506Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
483 | 2024-09-27T17:13:46.333Z | {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.804091084Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"0","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
484 | 2024-09-27T17:13:46.333Z | {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.804111583Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"1","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
485 | 2024-09-27T17:13:46.333Z | {"msg":"Transition from Reconcile to Active","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.804139142Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"client":"2","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
486 | 2024-09-27T17:13:46.333Z | {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.804157601Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
487 | 2024-09-27T17:13:46.333Z | {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.804184039Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
488 | 2024-09-27T17:13:46.333Z | Wait for a query_work_queue command to finish before sending IO |
489 | 2024-09-27T17:13:46.333Z | {"msg":"c7d40530-c950-4128-a645-c384a86d62f2 is now active with session: 7e4dc9d6-72a4-443b-80a5-97e066c91131","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.804212798Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
490 | 2024-09-27T17:13:46.333Z | {"msg":"The guest has finished waiting for activation with:33","v":0,"name":"crucible","level":30,"time":"2024-09-27T17:13:44.804238087Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230} |
491 | 2024-09-27T17:13:46.333Z | {"msg":"no Nexus client from DNS, aborting finish notification","v":0,"name":"crucible","level":50,"time":"2024-09-27T17:13:44.804253726Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230,"reconcile":"555c472f-4e08-494f-9fc2-8ab755cde469","":"downstairs","":"downstairs","session_id":"7e4dc9d6-72a4-443b-80a5-97e066c91131"} |
492 | 2024-09-27T17:13:46.333Z | Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800 |
493 | 2024-09-27T17:13:46.333Z | Loading write count information from file "/var/tmp/test_up-build/verify_file" |
494 | 2024-09-27T17:13:46.333Z | Read and Verify all blocks (0..50 range:true) |
495 | 2024-09-27T17:13:46.372Z | Wrote out file "/var/tmp/test_up-build/verify_file" |
496 | 2024-09-27T17:13:46.372Z | Verify test completed |
497 | 2024-09-27T17:13:46.375Z | Wrote out file "/var/tmp/test_up-build/verify_file" |
498 | 2024-09-27T17:13:46.375Z | CLIENT: Tests done. All submitted work has been ACK'd |
499 | 2024-09-27T17:13:46.375Z | ---------------------------------------------------------------- |
500 | 2024-09-27T17:13:46.375Z | Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:1 |
501 | 2024-09-27T17:13:46.375Z | States: Active Active Active |
502 | 2024-09-27T17:13:46.375Z | GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY |
503 | 2024-09-27T17:13:46.375Z | 1 Acked 1000 Read 50 Done Done Done false |
504 | 2024-09-27T17:13:46.375Z | STATES DS:0 DS:1 DS:2 TOTAL |
505 | 2024-09-27T17:13:46.375Z | Sent 0 0 0 0 |
506 | 2024-09-27T17:13:46.375Z | Done 1 1 1 3 |
507 | 2024-09-27T17:13:46.375Z | Skipped 0 0 0 0 |
508 | 2024-09-27T17:13:46.375Z | Error 0 0 0 0 |
509 | 2024-09-27T17:13:46.375Z | Last Flush: 0 0 0 |
510 | 2024-09-27T17:13:46.375Z | Downstairs last five completed: |
511 | 2024-09-27T17:13:46.375Z | Upstairs last five completed: 1 |
512 | 2024-09-27T17:13:46.376Z | CLIENT: Up:0 ds:1 act:3 |
513 | 2024-09-27T17:13:50.375Z | ---------------------------------------------------------------- |
514 | 2024-09-27T17:13:50.375Z | Crucible gen:33 GIO:true work queues: Upstairs:0 downstairs:0 |
515 | 2024-09-27T17:13:50.375Z | Downstairs last five completed: 1001 1000 |
516 | 2024-09-27T17:13:50.375Z | Upstairs last five completed: 2 1 |
517 | 2024-09-27T17:13:50.375Z | CLIENT: Up:0 ds:0 act:3 |
518 | 2024-09-27T17:13:50.375Z | CLIENT: All crucible jobs finished, exiting program |
519 | 2024-09-27T17:13:50.378Z | {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-09-27T17:13:48.847491305Z","hostname":"w-01J8T79TZ98ARZXF9AJ1D7W0D7","pid":1230} |
520 | 2024-09-27T17:13:50.381Z | Repair part 2 passed |
521 | 2024-09-27T17:13:50.384Z | /var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 |
522 | 2024-09-27T17:13:50.397Z | EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF |
523 | 2024-09-27T17:13:50.397Z | 0 000-009 32 32 32 1690 1690 1690 F F F |
524 | 2024-09-27T17:13:50.398Z | 1 010-019 32 32 32 1690 1690 1690 F F F |
525 | 2024-09-27T17:13:50.398Z | 2 020-029 32 32 32 1690 1690 1690 F F F |
526 | 2024-09-27T17:13:50.398Z | 3 030-039 31 31 31 1689 1689 1689 F F F |
527 | 2024-09-27T17:13:50.398Z | 4 040-049 32 32 32 1690 1690 1690 F F F |
528 | 2024-09-27T17:13:50.398Z | Max gen: 32, Max flush: 1690 |
529 | 2024-09-27T17:13:50.401Z | dump test passed |
530 | 2024-09-27T17:13:50.403Z | /var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -e 1 |
531 | 2024-09-27T17:13:50.416Z | Extent 1 |
532 | 2024-09-27T17:13:50.416Z | GEN 32 32 32 |
533 | 2024-09-27T17:13:50.416Z | FLUSH_ID 1690 1690 1690 |
534 | 2024-09-27T17:13:50.416Z | DIRTY |
535 | 2024-09-27T17:13:50.418Z | |
536 | 2024-09-27T17:13:50.418Z | BLOCK D0 D1 D2 C0 C1 C2 DIFF |
537 | 2024-09-27T17:13:50.418Z | 010 A A A A A A |
538 | 2024-09-27T17:13:50.421Z | 011 A A A A A A |
539 | 2024-09-27T17:13:50.421Z | 012 A A A A A A |
540 | 2024-09-27T17:13:50.424Z | 013 A A A A A A |
541 | 2024-09-27T17:13:50.424Z | 014 A A A A A A |
542 | 2024-09-27T17:13:50.424Z | 015 A A A A A A |
543 | 2024-09-27T17:13:50.426Z | 016 A A A A A A |
544 | 2024-09-27T17:13:50.426Z | 017 A A A A A A |
545 | 2024-09-27T17:13:50.429Z | 018 A A A A A A |
546 | 2024-09-27T17:13:50.429Z | 019 A A A A A A |
547 | 2024-09-27T17:13:50.432Z | dump extent test passed |
548 | 2024-09-27T17:13:50.432Z | /var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up-build/8810 -d /var/tmp/test_up-build/8820 -d /var/tmp/test_up-build/8830 -b 20 |
549 | 2024-09-27T17:13:50.447Z | Extent 2 Block in extent 0 Actual block 20 |
550 | 2024-09-27T17:13:50.447Z | |
551 | 2024-09-27T17:13:50.450Z | DATA SHA256 VER |
552 | 2024-09-27T17:13:50.450Z | ------ ---------------------------------------------------------------- --- |
553 | 2024-09-27T17:13:50.450Z | 0 06a4be7b2fd68b115515eff7147341bceb61feb4e1a511cefbd4a2ad91095ab2 A |
554 | 2024-09-27T17:13:50.450Z | 1 06a4be7b2fd68b115515eff7147341bceb61feb4e1a511cefbd4a2ad91095ab2 A |
555 | 2024-09-27T17:13:50.450Z | 2 06a4be7b2fd68b115515eff7147341bceb61feb4e1a511cefbd4a2ad91095ab2 A |
556 | 2024-09-27T17:13:50.450Z | |
557 | 2024-09-27T17:13:50.450Z | NONCES 0 1 2 DIFF |
558 | 2024-09-27T17:13:50.450Z | ------ ------------------------ ------------------------ ------------------------ ----- |
559 | 2024-09-27T17:13:50.450Z | 0 2cc9e8bf7a710ec527c9a4e7 2cc9e8bf7a710ec527c9a4e7 2cc9e8bf7a710ec527c9a4e7 |
560 | 2024-09-27T17:13:50.450Z | |
561 | 2024-09-27T17:13:50.450Z | TAGS 0 1 2 DIFF |
562 | 2024-09-27T17:13:50.450Z | ------ -------------------------------- -------------------------------- -------------------------------- ----- |
563 | 2024-09-27T17:13:50.453Z | 0 681e71f20eee1603c1c5d51bfae08f72 681e71f20eee1603c1c5d51bfae08f72 681e71f20eee1603c1c5d51bfae08f72 |
564 | 2024-09-27T17:13:50.453Z | |
565 | 2024-09-27T17:13:50.453Z | HASHES 0 1 2 DIFF |
566 | 2024-09-27T17:13:50.453Z | ------ ---------------- ---------------- ---------------- ----- |
567 | 2024-09-27T17:13:50.453Z | |
568 | 2024-09-27T17:13:50.455Z | dump block test passed |
569 | 2024-09-27T17:13:50.455Z | Upstairs tests have completed, stopping all downstairs |
570 | 2024-09-27T17:13:55.518Z | Creating four larger downstairs regions |
571 | 2024-09-27T17:13:55.957Z | Starting four downstairs |
572 | 2024-09-27T17:14:00.965Z | Now do the replace-reconcile test |
573 | 2024-09-27T17:17:39.238Z | Completed test: replace-reconcile |
574 | 2024-09-27T17:17:39.240Z | Now do the replace-before-active test |
575 | 2024-09-27T17:20:24.732Z | Completed test: replace-before-active |
576 | 2024-09-27T17:20:24.735Z | Tests have completed, stopping all downstairs |
577 | 2024-09-27T17:20:24.789Z | |
578 | 2024-09-27T17:20:24.789Z | All Tests have passed |
579 | 2024-09-27T17:20:24.792Z | 10:18 Test duration |
580 | 2024-09-27T17:20:24.792Z | |
581 | 2024-09-27T17:20:24.792Z | real 10:17.444051008 |
582 | 2024-09-27T17:20:24.792Z | user 3:53.341799684 |
583 | 2024-09-27T17:20:24.792Z | sys 13.700058410 |
584 | 2024-09-27T17:20:24.792Z | trap 0.206257022 |
585 | 2024-09-27T17:20:24.792Z | tflt 0.016944615 |
586 | 2024-09-27T17:20:24.792Z | dflt 0.053885078 |
587 | 2024-09-27T17:20:24.792Z | kflt 0.001094626 |
588 | 2024-09-27T17:20:24.792Z | lock 8:40:05.696392533 |
589 | 2024-09-27T17:20:24.792Z | slp 34:53.162453739 |
590 | 2024-09-27T17:20:24.792Z | lat 12.365948291 |
591 | 2024-09-27T17:20:24.792Z | stop 0.137402507 |
592 | 2024-09-27T17:20:24.792Z | + echo 'test-up-encrypted ends' |
593 | 2024-09-27T17:20:24.792Z | test-up-encrypted ends |
594 | 2024-09-27T17:20:29.796Z | process exited: duration 625783 ms, exit code 0 |
595 | 2024-09-27T17:20:29.796Z | exec warning: : stdio descriptors remain open after task exit; waiting 60 seconds for them to close |
596 | 2024-09-27T17:21:29.844Z | exec warning: : stdout descriptor may be held open by a background process; giving up! |
597 | 2024-09-27T17:21:29.844Z | exec warning: : stderr descriptor may be held open by a background process; giving up! |
|
598 | 2024-09-27T17:21:29.855Z | found 8 output files |
599 | 2024-09-27T17:21:29.855Z | uploading: /tmp/debug/df.txt (1270 bytes) |
600 | 2024-09-27T17:21:30.865Z | uploaded: /tmp/debug/df.txt |
601 | 2024-09-27T17:21:30.865Z | uploading: /tmp/debug/dtrace.txt (593345 bytes) |
602 | 2024-09-27T17:21:31.888Z | uploaded: /tmp/debug/dtrace.txt |
603 | 2024-09-27T17:21:31.891Z | uploading: /tmp/debug/iostat.txt (206330 bytes) |
604 | 2024-09-27T17:21:31.898Z | upload warning: file "/tmp/debug/iostat.txt" changed size mid upload: 206330 -> 206934 |
605 | 2024-09-27T17:21:32.904Z | uploaded: /tmp/debug/iostat.txt |
606 | 2024-09-27T17:21:32.907Z | uploading: /tmp/debug/mpstat.txt (499278 bytes) |
607 | 2024-09-27T17:21:32.919Z | upload warning: file "/tmp/debug/mpstat.txt" changed size mid upload: 499278 -> 501471 |
608 | 2024-09-27T17:21:33.926Z | uploaded: /tmp/debug/mpstat.txt |
609 | 2024-09-27T17:21:33.928Z | uploading: /tmp/debug/paging.txt (90813 bytes) |
610 | 2024-09-27T17:21:33.934Z | upload warning: file "/tmp/debug/paging.txt" changed size mid upload: 90813 -> 91281 |
611 | 2024-09-27T17:21:34.976Z | uploaded: /tmp/debug/paging.txt |
612 | 2024-09-27T17:21:34.976Z | uploading: /tmp/debug/prstat.txt (962976 bytes) |
613 | 2024-09-27T17:21:35.055Z | upload warning: file "/tmp/debug/prstat.txt" changed size mid upload: 962976 -> 969471 |
614 | 2024-09-27T17:21:36.115Z | uploaded: /tmp/debug/prstat.txt |
615 | 2024-09-27T17:21:36.115Z | uploading: /tmp/debug/psrinfo.txt (1528 bytes) |
616 | 2024-09-27T17:21:37.125Z | uploaded: /tmp/debug/psrinfo.txt |
617 | 2024-09-27T17:21:37.129Z | uploading: /tmp/debug/upstairs-info.txt (103174 bytes) |
618 | 2024-09-27T17:21:38.146Z | uploaded: /tmp/debug/upstairs-info.txt |