|
|
|
1 | 2023-08-10T05:59:03.690Z | job dependencies complete; ready to run (waiting for 22 m 29 s) |
2 | 2023-08-10T06:00:23.830Z | job assigned to worker 01H7F0E7GTC8H88H6D0K5G3VSY (queued for 1 m 20 s) |
3 | 2023-08-10T06:00:24.029Z | downloading input: /input/build/work/bins/crucible-downstairs.gz |
4 | 2023-08-10T06:00:37.865Z | downloaded input: /input/build/work/bins/crucible-downstairs.gz |
5 | 2023-08-10T06:00:37.879Z | downloading input: /input/build/work/bins/crucible-hammer.gz |
6 | 2023-08-10T06:00:48.742Z | downloaded input: /input/build/work/bins/crucible-hammer.gz |
7 | 2023-08-10T06:00:48.758Z | downloading input: /input/build/work/bins/crutest.gz |
8 | 2023-08-10T06:01:02.234Z | downloaded input: /input/build/work/bins/crutest.gz |
9 | 2023-08-10T06:01:02.238Z | downloading input: /input/build/work/bins/dsc.gz |
10 | 2023-08-10T06:01:09.326Z | downloaded input: /input/build/work/bins/dsc.gz |
11 | 2023-08-10T06:01:09.329Z | downloading input: /input/build/work/scripts/test_ds.sh |
12 | 2023-08-10T06:01:09.341Z | downloaded input: /input/build/work/scripts/test_ds.sh |
13 | 2023-08-10T06:01:09.344Z | downloading input: /input/build/work/scripts/test_live_repair.sh |
14 | 2023-08-10T06:01:09.357Z | downloaded input: /input/build/work/scripts/test_live_repair.sh |
15 | 2023-08-10T06:01:09.360Z | downloading input: /input/build/work/scripts/test_repair.sh |
16 | 2023-08-10T06:01:09.375Z | downloaded input: /input/build/work/scripts/test_repair.sh |
17 | 2023-08-10T06:01:09.378Z | downloading input: /input/build/work/scripts/test_up.sh |
18 | 2023-08-10T06:01:09.393Z | downloaded input: /input/build/work/scripts/test_up.sh |
19 | 2023-08-10T06:01:09.397Z | downloading input: /input/build/tmp/cargo-test-out.log |
20 | 2023-08-10T06:01:10.321Z | downloaded input: /input/build/tmp/cargo-test-out.log |
|
21 | 2023-08-10T06:01:10.325Z | starting task 0: "setup" |
22 | 2023-08-10T06:01:10.459Z | ++ uname -s |
23 | 2023-08-10T06:01:10.462Z | + kern=SunOS |
24 | 2023-08-10T06:01:10.465Z | + case "$kern" in |
25 | 2023-08-10T06:01:10.468Z | + groupadd -g 12345 build |
26 | 2023-08-10T06:01:10.471Z | + useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build |
27 | 2023-08-10T06:01:12.477Z | + zfs create -o mountpoint=/work rpool/work |
28 | 2023-08-10T06:01:14.106Z | ++ awk '$2 == "/home" { print $3 }' /etc/mnttab |
29 | 2023-08-10T06:01:14.110Z | + home_fs=zfs |
30 | 2023-08-10T06:01:14.112Z | + [[ zfs == autofs ]] |
31 | 2023-08-10T06:01:14.115Z | + mkdir -p /home/build |
32 | 2023-08-10T06:01:14.118Z | + chown build:build /home/build /work |
33 | 2023-08-10T06:01:14.121Z | + chmod 0700 /home/build /work |
34 | 2023-08-10T06:01:14.124Z | process exited: duration 3791 ms, exit code 0 |
|
35 | 2023-08-10T06:01:14.130Z | starting task 1: "authentication" |
36 | 2023-08-10T06:01:14.159Z | process exited: duration 25 ms, exit code 0 |
|
37 | 2023-08-10T06:01:14.165Z | starting task 2: "build" |
38 | 2023-08-10T06:01:14.175Z | + echo 'input bins dir contains:' |
39 | 2023-08-10T06:01:14.178Z | input bins dir contains: |
40 | 2023-08-10T06:01:14.180Z | + ls -ltr /input/build/work/bins |
41 | 2023-08-10T06:01:14.183Z | total 673744 |
42 | 2023-08-10T06:01:14.186Z | -rw-r--r-- 1 root root 108899860 Aug 10 06:00 crucible-downstairs.gz |
43 | 2023-08-10T06:01:14.189Z | -rw-r--r-- 1 root root 69594543 Aug 10 06:00 crucible-hammer.gz |
44 | 2023-08-10T06:01:14.192Z | -rw-r--r-- 1 root root 97531181 Aug 10 06:01 crutest.gz |
45 | 2023-08-10T06:01:14.195Z | -rw-r--r-- 1 root root 68703972 Aug 10 06:01 dsc.gz |
46 | 2023-08-10T06:01:14.198Z | + banner unpack |
47 | 2023-08-10T06:01:14.241Z | |
48 | 2023-08-10T06:01:14.244Z | # # # # ##### ## #### # # |
49 | 2023-08-10T06:01:14.248Z | # # ## # # # # # # # # # |
50 | 2023-08-10T06:01:14.251Z | # # # # # # # # # # #### |
51 | 2023-08-10T06:01:14.253Z | # # # # # ##### ###### # # # |
52 | 2023-08-10T06:01:14.256Z | # # # ## # # # # # # # |
53 | 2023-08-10T06:01:14.259Z | #### # # # # # #### # # |
54 | 2023-08-10T06:01:14.262Z | |
55 | 2023-08-10T06:01:14.265Z | + mkdir -p /var/tmp/bins |
56 | 2023-08-10T06:01:14.268Z | + for t in "$input/bins/"*.gz |
57 | 2023-08-10T06:01:14.271Z | ++ basename /input/build/work/bins/crucible-downstairs.gz |
58 | 2023-08-10T06:01:14.273Z | + b=crucible-downstairs.gz |
59 | 2023-08-10T06:01:14.276Z | + b=crucible-downstairs |
60 | 2023-08-10T06:01:14.279Z | + gunzip |
61 | 2023-08-10T06:01:18.055Z | + chmod +x /var/tmp/bins/crucible-downstairs |
62 | 2023-08-10T06:01:18.058Z | + for t in "$input/bins/"*.gz |
63 | 2023-08-10T06:01:18.061Z | ++ basename /input/build/work/bins/crucible-hammer.gz |
64 | 2023-08-10T06:01:18.063Z | + b=crucible-hammer.gz |
65 | 2023-08-10T06:01:18.066Z | + b=crucible-hammer |
66 | 2023-08-10T06:01:18.069Z | + gunzip |
67 | 2023-08-10T06:01:20.527Z | + chmod +x /var/tmp/bins/crucible-hammer |
68 | 2023-08-10T06:01:20.530Z | + for t in "$input/bins/"*.gz |
69 | 2023-08-10T06:01:20.533Z | ++ basename /input/build/work/bins/crutest.gz |
70 | 2023-08-10T06:01:20.536Z | + b=crutest.gz |
71 | 2023-08-10T06:01:20.539Z | + b=crutest |
72 | 2023-08-10T06:01:20.542Z | + gunzip |
73 | 2023-08-10T06:01:23.861Z | + chmod +x /var/tmp/bins/crutest |
74 | 2023-08-10T06:01:23.864Z | + for t in "$input/bins/"*.gz |
75 | 2023-08-10T06:01:23.867Z | ++ basename /input/build/work/bins/dsc.gz |
76 | 2023-08-10T06:01:23.869Z | + b=dsc.gz |
77 | 2023-08-10T06:01:23.872Z | + b=dsc |
78 | 2023-08-10T06:01:23.875Z | + gunzip |
79 | 2023-08-10T06:01:26.263Z | + chmod +x /var/tmp/bins/dsc |
80 | 2023-08-10T06:01:26.266Z | + export BINDIR=/var/tmp/bins |
81 | 2023-08-10T06:01:26.269Z | + BINDIR=/var/tmp/bins |
82 | 2023-08-10T06:01:26.272Z | + banner test_up_encrypted |
83 | 2023-08-10T06:01:26.275Z | |
84 | 2023-08-10T06:01:26.278Z | ##### ###### #### ##### # # ##### ###### # # |
85 | 2023-08-10T06:01:26.281Z | # # # # # # # # # ## # |
86 | 2023-08-10T06:01:26.284Z | # ##### #### # # # # # ##### # # # |
87 | 2023-08-10T06:01:26.287Z | # # # # # # ##### # # # # |
88 | 2023-08-10T06:01:26.290Z | # # # # # # # # # # ## |
89 | 2023-08-10T06:01:26.293Z | # ###### #### # ####### #### # ####### ###### # # |
90 | 2023-08-10T06:01:26.296Z | |
91 | 2023-08-10T06:01:26.299Z | + ptime -m bash /input/build/work/scripts/test_up.sh -N encrypted |
92 | 2023-08-10T06:01:26.302Z | /input/build/work |
93 | 2023-08-10T06:01:26.304Z | Turn off color for downstairs dump |
94 | 2023-08-10T06:01:26.459Z | Upstairs using key: uUNQZMLN6+7Z1bxrX+G3ctqq/MPXkUp3G36OWVCYmCQ= |
95 | 2023-08-10T06:01:26.462Z | dsc output goes to /tmp/test_up/dsc-out.txt |
96 | 2023-08-10T06:01:26.465Z | Creating three downstairs regions |
97 | 2023-08-10T06:01:26.876Z | Starting three downstairs |
98 | 2023-08-10T06:01:31.883Z | Disable automatic restart on all downstairs |
99 | 2023-08-10T06:01:31.999Z | |
100 | 2023-08-10T06:01:32.002Z | Begin tests, output goes to /tmp/test_up/test_up_out.txt |
101 | 2023-08-10T06:01:32.005Z | Running test: span |
102 | 2023-08-10T06:01:36.159Z | Completed test: span |
103 | 2023-08-10T06:01:36.163Z | Running test: big |
104 | 2023-08-10T06:01:40.923Z | Completed test: big |
105 | 2023-08-10T06:01:40.927Z | Running test: dep |
106 | 2023-08-10T06:03:33.132Z | Completed test: dep |
107 | 2023-08-10T06:03:33.136Z | Running test: balloon |
108 | 2023-08-10T06:06:03.295Z | Completed test: balloon |
109 | 2023-08-10T06:06:03.299Z | Running test: deactivate |
110 | 2023-08-10T06:06:28.655Z | Completed test: deactivate |
111 | 2023-08-10T06:06:28.660Z | Running hammer |
112 | 2023-08-10T06:06:57.145Z | Run repair tests |
113 | 2023-08-10T06:06:57.149Z | /var/tmp/bins/crutest fill -g 26 -q --key uUNQZMLN6+7Z1bxrX+G3ctqq/MPXkUp3G36OWVCYmCQ= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file |
114 | 2023-08-10T06:06:57.170Z | {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.279369693Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
115 | 2023-08-10T06:06:57.175Z | {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f9f88bb1717d2b86fef41798471b705ff437b87\",\n git_commit_timestamp: \"2023-08-10T05:35:49.000000000Z\",\n git_branch: \"alan/wip-for-vcr\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.279786503Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
116 | 2023-08-10T06:06:57.177Z | {"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.279838765Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
117 | 2023-08-10T06:06:57.180Z | {"msg":"Crucible stats registered with UUID: 88c1d1d4-5443-42be-8500-ffdf9afad768","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.279862822Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
118 | 2023-08-10T06:06:57.183Z | {"msg":"Crucible 88c1d1d4-5443-42be-8500-ffdf9afad768 has session id: 84247556-58be-425d-8f7a-6366880ae9d4","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.27988502Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
119 | 2023-08-10T06:06:57.185Z | Crucible runtime is spawned |
120 | 2023-08-10T06:06:57.188Z | The guest has requested activation with gen:26 |
121 | 2023-08-10T06:06:57.191Z | {"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.280461839Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188,"task":"up_listen"} |
122 | 2023-08-10T06:06:57.193Z | {"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.280542577Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
123 | 2023-08-10T06:06:57.196Z | {"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.280607651Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
124 | 2023-08-10T06:06:57.199Z | {"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.28067326Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188,"looper":"1"} |
125 | 2023-08-10T06:06:57.201Z | {"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.280878462Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188,"looper":"0"} |
126 | 2023-08-10T06:06:57.204Z | {"msg":"[1] 88c1d1d4-5443-42be-8500-ffdf9afad768 looper connected","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.280970223Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188,"looper":"1"} |
127 | 2023-08-10T06:06:57.207Z | {"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.281034716Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188,"looper":"2"} |
128 | 2023-08-10T06:06:57.210Z | {"msg":"[0] 88c1d1d4-5443-42be-8500-ffdf9afad768 looper connected","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.281100201Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188,"looper":"0"} |
129 | 2023-08-10T06:06:57.213Z | {"msg":"[2] 88c1d1d4-5443-42be-8500-ffdf9afad768 looper connected","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.281199587Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188,"looper":"2"} |
130 | 2023-08-10T06:06:57.216Z | {"msg":"88c1d1d4-5443-42be-8500-ffdf9afad768 active request set","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.281661949Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
131 | 2023-08-10T06:06:57.218Z | {"msg":"Set desired generation to :26","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.281731974Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
132 | 2023-08-10T06:06:57.221Z | {"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.281922284Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
133 | 2023-08-10T06:06:57.224Z | {"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.282209537Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
134 | 2023-08-10T06:06:57.226Z | {"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.282261754Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
135 | 2023-08-10T06:06:57.229Z | {"msg":"[1] 88c1d1d4-5443-42be-8500-ffdf9afad768 (56c24d0d-910f-48eb-a997-1aae37399f7c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.282565737Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
136 | 2023-08-10T06:06:57.232Z | {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.282630805Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
137 | 2023-08-10T06:06:57.234Z | {"msg":"[1] client is_active_req TRUE, promote! session 56c24d0d-910f-48eb-a997-1aae37399f7c","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.282678787Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
138 | 2023-08-10T06:06:57.237Z | {"msg":"[2] 88c1d1d4-5443-42be-8500-ffdf9afad768 (56c24d0d-910f-48eb-a997-1aae37399f7c) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.282796775Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
139 | 2023-08-10T06:06:57.240Z | {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.282939208Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
140 | 2023-08-10T06:06:57.243Z | {"msg":"[2] client is_active_req TRUE, promote! session 56c24d0d-910f-48eb-a997-1aae37399f7c","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283017809Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
141 | 2023-08-10T06:06:57.245Z | {"msg":"[0] 88c1d1d4-5443-42be-8500-ffdf9afad768 (56c24d0d-910f-48eb-a997-1aae37399f7c) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283092748Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
142 | 2023-08-10T06:06:57.248Z | {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283168046Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
143 | 2023-08-10T06:06:57.251Z | {"msg":"[0] client is_active_req TRUE, promote! session 56c24d0d-910f-48eb-a997-1aae37399f7c","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283245627Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
144 | 2023-08-10T06:06:57.254Z | {"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283496072Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
145 | 2023-08-10T06:06:57.257Z | {"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283564125Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
146 | 2023-08-10T06:06:57.260Z | {"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008820, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283616443Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
147 | 2023-08-10T06:06:57.262Z | {"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008830, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283667077Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
148 | 2023-08-10T06:06:57.265Z | {"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283722551Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
149 | 2023-08-10T06:06:57.268Z | {"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008810, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283748965Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
150 | 2023-08-10T06:06:57.270Z | {"msg":"88c1d1d4-5443-42be-8500-ffdf9afad768 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283789597Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
151 | 2023-08-10T06:06:57.273Z | {"msg":"88c1d1d4-5443-42be-8500-ffdf9afad768 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283834735Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
152 | 2023-08-10T06:06:57.276Z | {"msg":"88c1d1d4-5443-42be-8500-ffdf9afad768 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.283901462Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
153 | 2023-08-10T06:06:57.278Z | {"msg":"[1] 88c1d1d4-5443-42be-8500-ffdf9afad768 (56c24d0d-910f-48eb-a997-1aae37399f7c) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.284328371Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
154 | 2023-08-10T06:06:57.281Z | {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.284387328Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
155 | 2023-08-10T06:06:57.283Z | {"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-08-10T06:06:57.284413039Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
156 | 2023-08-10T06:06:57.286Z | {"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.28448613Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
157 | 2023-08-10T06:06:57.289Z | {"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.284592404Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
158 | 2023-08-10T06:06:57.292Z | {"msg":"88c1d1d4-5443-42be-8500-ffdf9afad768 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.284683396Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
159 | 2023-08-10T06:06:57.294Z | {"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.284756966Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
160 | 2023-08-10T06:06:57.297Z | {"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.284805585Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
161 | 2023-08-10T06:06:57.300Z | {"msg":"[2] 88c1d1d4-5443-42be-8500-ffdf9afad768 (56c24d0d-910f-48eb-a997-1aae37399f7c) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.2848492Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
162 | 2023-08-10T06:06:57.303Z | {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.284891813Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
163 | 2023-08-10T06:06:57.305Z | {"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-08-10T06:06:57.284931059Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
164 | 2023-08-10T06:06:57.308Z | {"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.284970141Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
165 | 2023-08-10T06:06:57.311Z | {"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285010853Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
166 | 2023-08-10T06:06:57.314Z | {"msg":"88c1d1d4-5443-42be-8500-ffdf9afad768 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285052186Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
167 | 2023-08-10T06:06:57.316Z | {"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285099399Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
168 | 2023-08-10T06:06:57.319Z | {"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.2851201Z","The guest has finished waiting for activation with:26 |
169 | 2023-08-10T06:06:57.324Z | Wait for a query_work_queue command to finish before sending IO |
170 | 2023-08-10T06:06:57.327Z | hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
171 | 2023-08-10T06:06:57.330Z | {"msg":"[0] 88c1d1d4-5443-42be-8500-ffdf9afad768 (56c24d0d-910f-48eb-a997-1aae37399f7c) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285227663Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
172 | 2023-08-10T06:06:57.333Z | {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285257093Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
173 | 2023-08-10T06:06:57.335Z | {"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-08-10T06:06:57.285286901Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
174 | 2023-08-10T06:06:57.338Z | {"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285374971Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
175 | 2023-08-10T06:06:57.341Z | {"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285413956Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
176 | 2023-08-10T06:06:57.343Z | {"msg":"88c1d1d4-5443-42be-8500-ffdf9afad768 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285473172Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
177 | 2023-08-10T06:06:57.346Z | {"msg":"[0]R flush_numbers: [2015, 2016, 2016, 2016, 2016]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285531841Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
178 | 2023-08-10T06:06:57.349Z | {"msg":"[0]R generation: [20, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.28559538Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
179 | 2023-08-10T06:06:57.351Z | {"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285657323Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
180 | 2023-08-10T06:06:57.354Z | {"msg":"[1]R flush_numbers: [2015, 2016, 2016, 2016, 2016]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.28571811Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
181 | 2023-08-10T06:06:57.357Z | {"msg":"[1]R generation: [20, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285786204Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
182 | 2023-08-10T06:06:57.360Z | {"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285817379Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
183 | 2023-08-10T06:06:57.362Z | {"msg":"[2]R flush_numbers: [2015, 2016, 2016, 2016, 2016]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285838332Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
184 | 2023-08-10T06:06:57.365Z | {"msg":"[2]R generation: [20, 20, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285857139Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
185 | 2023-08-10T06:06:57.368Z | {"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285876165Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
186 | 2023-08-10T06:06:57.371Z | {"msg":"Max found gen is 21","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285894669Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
187 | 2023-08-10T06:06:57.375Z | {"msg":"Generation requested: 26 >= found:21","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285912938Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
188 | 2023-08-10T06:06:57.378Z | {"msg":"Next flush: 2017","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285930743Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
189 | 2023-08-10T06:06:57.380Z | {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285949318Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
190 | 2023-08-10T06:06:57.383Z | {"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285967897Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
191 | 2023-08-10T06:06:57.386Z | {"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.285986948Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
192 | 2023-08-10T06:06:57.388Z | {"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.286005419Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
193 | 2023-08-10T06:06:57.391Z | {"msg":"88c1d1d4-5443-42be-8500-ffdf9afad768 is now active with session: 56c24d0d-910f-48eb-a997-1aae37399f7c","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.286025742Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
194 | 2023-08-10T06:06:57.394Z | Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800 |
195 | 2023-08-10T06:06:57.396Z | {"msg":"88c1d1d4-5443-42be-8500-ffdf9afad768 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.28604915Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
196 | 2023-08-10T06:06:57.399Z | {"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.286068518Z","hostname":"Fill test |
197 | 2023-08-10T06:06:57.402Z | ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
198 | 2023-08-10T06:06:57.404Z | {"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.286088734Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
199 | 2023-08-10T06:06:57.408Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.286107016Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
200 | 2023-08-10T06:06:57.411Z | {"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.286125965Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
201 | 2023-08-10T06:06:57.414Z | {"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.286155814Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
202 | 2023-08-10T06:06:57.416Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.28617858Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
203 | 2023-08-10T06:06:57.419Z | {"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.28619809Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
204 | 2023-08-10T06:06:57.422Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.286216828Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
205 | 2023-08-10T06:06:57.425Z | {"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.28623546Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
206 | 2023-08-10T06:06:57.427Z | {"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.286254264Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
207 | 2023-08-10T06:06:57.430Z | {"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:57.286272916Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
208 | 2023-08-10T06:06:57.433Z | Read and Verify all blocks (0..50 range:false) |
209 | 2023-08-10T06:06:57.435Z | Wrote out file "/var/tmp/test_up/verify_file" |
210 | 2023-08-10T06:06:57.438Z | CLIENT: Tests done. All submitted work has been ACK'd |
211 | 2023-08-10T06:06:57.457Z | ---------------------------------------------------------------- |
212 | 2023-08-10T06:06:57.460Z | Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:1 |
213 | 2023-08-10T06:06:57.463Z | GW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY |
214 | 2023-08-10T06:06:57.465Z | 3 Acked 1002 Read 50 Done Done Done false |
215 | 2023-08-10T06:06:57.468Z | STATES DS:0 DS:1 DS:2 TOTAL |
216 | 2023-08-10T06:06:57.471Z | New 0 0 0 0 |
217 | 2023-08-10T06:06:57.473Z | Sent 0 0 0 0 |
218 | 2023-08-10T06:06:57.476Z | Done 1 1 1 3 |
219 | 2023-08-10T06:06:57.479Z | Skipped 0 0 0 0 |
220 | 2023-08-10T06:06:57.481Z | Error 0 0 0 0 |
221 | 2023-08-10T06:06:57.484Z | Last Flush: 1001 1001 1001 |
222 | 2023-08-10T06:06:57.487Z | Downstairs last five completed: 1001 1000 |
223 | 2023-08-10T06:06:57.489Z | Upstairs last five completed: 3 2 1 |
224 | 2023-08-10T06:06:57.492Z | CLIENT: Up:0 ds:1 act:3 |
225 | 2023-08-10T06:06:58.176Z | {"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:58.286452756Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
226 | 2023-08-10T06:06:58.179Z | {"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:06:58.286551743Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1188} |
227 | 2023-08-10T06:07:01.457Z | ---------------------------------------------------------------- |
228 | 2023-08-10T06:07:01.460Z | Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:0 |
229 | 2023-08-10T06:07:01.463Z | Downstairs last five completed: 1003 1002 1001 1000 |
230 | 2023-08-10T06:07:01.467Z | Upstairs last five completed: 4 3 2 1 |
231 | 2023-08-10T06:07:01.470Z | CLIENT: Up:0 ds:0 act:3 |
232 | 2023-08-10T06:07:01.473Z | CLIENT: All crucible jobs finished, exiting program |
233 | 2023-08-10T06:07:01.476Z | Repair setup passed |
234 | 2023-08-10T06:07:01.480Z | Copy the 8830 file |
235 | 2023-08-10T06:07:01.483Z | cp -r /var/tmp/test_up/8830 /var/tmp/test_up/previous |
236 | 2023-08-10T06:07:01.509Z | /var/tmp/bins/crutest repair -g 27 -q --key uUNQZMLN6+7Z1bxrX+G3ctqq/MPXkUp3G36OWVCYmCQ= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file |
237 | 2023-08-10T06:07:01.531Z | {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.642330054Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
238 | 2023-08-10T06:07:01.534Z | {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f9f88bb1717d2b86fef41798471b705ff437b87\",\n git_commit_timestamp: \"2023-08-10T05:35:49.000000000Z\",\n git_branch: \"alan/wip-for-vcr\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.642839761Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
239 | 2023-08-10T06:07:01.537Z | {"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.642892854Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
240 | 2023-08-10T06:07:01.541Z | {"msg":"Crucible stats registered with UUID: 3a64b8d7-0361-4868-8162-bc470ea1c9e8","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.642919071Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
241 | 2023-08-10T06:07:01.544Z | {"msg":"Crucible 3a64b8d7-0361-4868-8162-bc470ea1c9e8 has session id: 72aaf95f-8aa1-4238-9fe0-5b7a203150f2","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.642988691Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
242 | 2023-08-10T06:07:01.547Z | Crucible runtime is spawned |
243 | 2023-08-10T06:07:01.550Z | The guest has requested activation with gen:27 |
244 | 2023-08-10T06:07:01.553Z | {"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.643396087Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194,"task":"up_listen"} |
245 | 2023-08-10T06:07:01.560Z | {"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.643465871Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194,"looper":"2"} |
246 | 2023-08-10T06:07:01.564Z | {"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.64352243Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
247 | 2023-08-10T06:07:01.567Z | {"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.64358605Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
248 | 2023-08-10T06:07:01.570Z | {"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.643700875Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194,"looper":"0"} |
249 | 2023-08-10T06:07:01.574Z | {"msg":"[2] 3a64b8d7-0361-4868-8162-bc470ea1c9e8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.643783671Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194,"looper":"2"} |
250 | 2023-08-10T06:07:01.577Z | {"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.643853745Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194,"looper":"1"} |
251 | 2023-08-10T06:07:01.580Z | {"msg":"[0] 3a64b8d7-0361-4868-8162-bc470ea1c9e8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.643933349Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194,"looper":"0"} |
252 | 2023-08-10T06:07:01.583Z | {"msg":"[1] 3a64b8d7-0361-4868-8162-bc470ea1c9e8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.643975382Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194,"looper":"1"} |
253 | 2023-08-10T06:07:01.586Z | {"msg":"3a64b8d7-0361-4868-8162-bc470ea1c9e8 active request set","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.644479933Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
254 | 2023-08-10T06:07:01.589Z | {"msg":"Set desired generation to :27","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.644533669Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
255 | 2023-08-10T06:07:01.592Z | {"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.644749712Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
256 | 2023-08-10T06:07:01.596Z | {"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.645059365Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
257 | 2023-08-10T06:07:01.599Z | {"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.645124996Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
258 | 2023-08-10T06:07:01.602Z | {"msg":"[2] 3a64b8d7-0361-4868-8162-bc470ea1c9e8 (55b0b733-dd8c-4330-a585-85db0d210c47) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.645462871Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
259 | 2023-08-10T06:07:01.605Z | {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.645522198Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
260 | 2023-08-10T06:07:01.608Z | {"msg":"[2] client is_active_req TRUE, promote! session 55b0b733-dd8c-4330-a585-85db0d210c47","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.645565435Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
261 | 2023-08-10T06:07:01.611Z | {"msg":"[1] 3a64b8d7-0361-4868-8162-bc470ea1c9e8 (55b0b733-dd8c-4330-a585-85db0d210c47) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.645714614Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
262 | 2023-08-10T06:07:01.614Z | {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.645781901Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
263 | 2023-08-10T06:07:01.617Z | {"msg":"[1] client is_active_req TRUE, promote! session 55b0b733-dd8c-4330-a585-85db0d210c47","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.645851932Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
264 | 2023-08-10T06:07:01.620Z | {"msg":"[0] 3a64b8d7-0361-4868-8162-bc470ea1c9e8 (55b0b733-dd8c-4330-a585-85db0d210c47) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.645888114Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
265 | 2023-08-10T06:07:01.623Z | {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.645922875Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
266 | 2023-08-10T06:07:01.626Z | {"msg":"[0] client is_active_req TRUE, promote! session 55b0b733-dd8c-4330-a585-85db0d210c47","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.64596268Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
267 | 2023-08-10T06:07:01.629Z | {"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.646336659Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
268 | 2023-08-10T06:07:01.632Z | {"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.646393997Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
269 | 2023-08-10T06:07:01.634Z | {"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008830, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.646461237Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
270 | 2023-08-10T06:07:01.639Z | {"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008820, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.646536462Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
271 | 2023-08-10T06:07:01.642Z | {"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.646576104Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
272 | 2023-08-10T06:07:01.645Z | {"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008810, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.646649383Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
273 | 2023-08-10T06:07:01.648Z | {"msg":"3a64b8d7-0361-4868-8162-bc470ea1c9e8 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.646843931Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
274 | 2023-08-10T06:07:01.651Z | {"msg":"3a64b8d7-0361-4868-8162-bc470ea1c9e8 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.646897669Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
275 | 2023-08-10T06:07:01.654Z | {"msg":"3a64b8d7-0361-4868-8162-bc470ea1c9e8 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.646922235Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
276 | 2023-08-10T06:07:01.657Z | {"msg":"[1] 3a64b8d7-0361-4868-8162-bc470ea1c9e8 (55b0b733-dd8c-4330-a585-85db0d210c47) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.647189389Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
277 | 2023-08-10T06:07:01.660Z | {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.647267595Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
278 | 2023-08-10T06:07:01.667Z | {"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-08-10T06:07:01.64732731Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
279 | 2023-08-10T06:07:01.671Z | {"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.647397533Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
280 | 2023-08-10T06:07:01.674Z | {"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.647454733Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
281 | 2023-08-10T06:07:01.677Z | {"msg":"3a64b8d7-0361-4868-8162-bc470ea1c9e8 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.64751236Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
282 | 2023-08-10T06:07:01.681Z | {"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.647567721Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
283 | 2023-08-10T06:07:01.684Z | {"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.647624143Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
284 | 2023-08-10T06:07:01.687Z | {"msg":"[0] 3a64b8d7-0361-4868-8162-bc470ea1c9e8 (55b0b733-dd8c-4330-a585-85db0d210c47) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.647683877Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
285 | 2023-08-10T06:07:01.690Z | {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.647738792Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
286 | 2023-08-10T06:07:01.693Z | {"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-08-10T06:07:01.647805265Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
287 | 2023-08-10T06:07:01.697Z | {"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.647894064Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
288 | 2023-08-10T06:07:01.700Z | {"msg":"[2] 3a64b8d7-0361-4868-8162-bc470ea1c9e8 (55b0b733-dd8c-4330-a585-85db0d210c47) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.647957801Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
289 | 2023-08-10T06:07:01.702Z | {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:27 |
290 | 2023-08-10T06:07:01.705Z | Wait for a query_work_queue command to finish before sending IO |
291 | 2023-08-10T06:07:01.709Z | ,"time":"2023-08-10T06:07:01.648074499Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
292 | 2023-08-10T06:07:01.712Z | {"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648195577Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
293 | 2023-08-10T06:07:01.716Z | {"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-08-10T06:07:01.648249843Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
294 | 2023-08-10T06:07:01.719Z | {"msg":"3a64b8d7-0361-4868-8162-bc470ea1c9e8 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648271885Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
295 | 2023-08-10T06:07:01.722Z | {"msg":"[1]R flush_numbers: [2017, 2017, 2017, 2017, 2017]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648314555Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
296 | 2023-08-10T06:07:01.725Z | {"msg":"[1]R generation: [26, 26, 26, 26, 26]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648359156Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
297 | 2023-08-10T06:07:01.728Z | {"msg":"[1]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648399149Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
298 | 2023-08-10T06:07:01.731Z | {"msg":"[2]R flush_numbers: [2017, 2017, 2017, 2017, 2017]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648439261Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
299 | 2023-08-10T06:07:01.736Z | {"msg":"[2]R generation: [26, 26, 26, 26, 26]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648483358Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
300 | 2023-08-10T06:07:01.740Z | {"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648525039Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
301 | 2023-08-10T06:07:01.743Z | {"msg":"[0]R flush_numbers: [2017, 2017, 2017, 2017, 2017]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648565075Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
302 | 2023-08-10T06:07:01.745Z | {"msg":"[0]R generation: [26, 26, 26, 26, 26]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648604844Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
303 | 2023-08-10T06:07:01.749Z | {"msg":"[0]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648648305Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
304 | 2023-08-10T06:07:01.752Z | {"msg":"Max found gen is 27","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648692246Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
305 | 2023-08-10T06:07:01.755Z | {"msg":"Generation requested: 27 >= found:27","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.64871256Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
306 | 2023-08-10T06:07:01.758Z | {"msg":"Next flush: 2018","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648731286Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
307 | 2023-08-10T06:07:01.761Z | {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648749977Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
308 | 2023-08-10T06:07:01.764Z | {"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648768693Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
309 | 2023-08-10T06:07:01.767Z | {"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.64878736Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
310 | 2023-08-10T06:07:01.770Z | {"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648806379Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
311 | 2023-08-10T06:07:01.773Z | {"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648825142Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
312 | 2023-08-10T06:07:01.776Z | {"msg":"3a64b8d7-0361-4868-8162-bc470ea1c9e8 is now active with session: 55b0b733-dd8c-4330-a585-85db0d210c47","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.64884629Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
313 | 2023-08-10T06:07:01.779Z | {"msg":"3a64b8d7-0361-4868-8162-bc470ea1c9e8 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648865995Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
314 | 2023-08-10T06:07:01.782Z | {"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648884698Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
315 | 2023-08-10T06:07:01.786Z | {"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648902645Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
316 | 2023-08-10T06:07:01.789Z | {"msg":"[2] 127.0.0.1:8830 task reports connection:true","vRegion: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800 |
317 | 2023-08-10T06:07:01.792Z | ":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648926545Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
318 | 2023-08-10T06:07:01.795Z | {"msg":"3a64b8d7-0361-4868-8162-bc470ea1c9e8 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648946096Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
319 | 2023-08-10T06:07:01.798Z | {"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648964536Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
320 | 2023-08-10T06:07:01.801Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.648984048Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
321 | 2023-08-10T06:07:01.804Z | {"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.649002921Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
322 | 2023-08-10T06:07:01.807Z | {"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.649021001Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
323 | 2023-08-10T06:07:01.810Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.649039985Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
324 | 2023-08-10T06:07:01.813Z | {"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.649058757Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
325 | 2023-08-10T06:07:01.817Z | {"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.649077123Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
326 | 2023-08-10T06:07:01.820Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.649096064Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
327 | 2023-08-10T06:07:01.823Z | {"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.649114266Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
328 | 2023-08-10T06:07:01.826Z | {"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:01.649147272Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1194} |
329 | 2023-08-10T06:07:01.829Z | Loading write count information from file "/var/tmp/test_up/verify_file" |
330 | 2023-08-10T06:07:01.832Z | Run Repair workload |
331 | 2023-08-10T06:07:01.835Z | 01/10 Write block 24 len 20480 data: 2 2 2 2 2 |
332 | 2023-08-10T06:07:01.838Z | 02/10 Write block 24 len 16384 data: 3 3 3 3 |
333 | 2023-08-10T06:07:01.841Z | 03/10 Read block 14 len 24576 |
334 | 2023-08-10T06:07:01.844Z | 04/10 Write block 14 len 32768 data: 2 2 2 2 2 2 2 2 |
335 | 2023-08-10T06:07:01.847Z | 05/10 Read block 14 len 16384 |
336 | 2023-08-10T06:07:01.850Z | 06/10 Read block 17 len 36864 |
337 | 2023-08-10T06:07:01.853Z | 07/10 Write block 40 len 12288 data: 2 2 2 |
338 | 2023-08-10T06:07:01.857Z | 08/10 Write block 22 len 20480 data: 2 2 4 4 4 |
339 | 2023-08-10T06:07:01.860Z | 09/10 Read block 2 len 8192 |
340 | 2023-08-10T06:07:01.863Z | 10/10 Write block 31 len 8192 data: 2 2 |
341 | 2023-08-10T06:07:01.865Z | ---------------------------------------------------------------- |
342 | 2023-08-10T06:07:01.869Z | Crucible gen:27 GIO:true work queues: Upstairs:0 downstairs:10 |
343 | 2023-08-10T06:07:01.872Z | GW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY |
344 | 2023-08-10T06:07:01.875Z | 1 Acked 1000 Write 5 Done Done Done false |
345 | 2023-08-10T06:07:01.878Z | 2 Acked 1001 Write 4 Done Done Done false |
346 | 2023-08-10T06:07:01.881Z | 3 Acked 1002 Read 6 Done Done Done false |
347 | 2023-08-10T06:07:01.884Z | 4 Acked 1003 Write 8 Done Done Done false |
348 | 2023-08-10T06:07:01.888Z | 5 Acked 1004 Read 4 Done Done Done false |
349 | 2023-08-10T06:07:01.891Z | 6 Acked 1005 Read 9 Done Done Done false |
350 | 2023-08-10T06:07:01.894Z | 7 Acked 1006 Write 3 Done Done Done false |
351 | 2023-08-10T06:07:01.897Z | 8 Acked 1007 Write 5 Done Done Done false |
352 | 2023-08-10T06:07:01.900Z | 9 Acked 1008 Read 2 Done Done Done false |
353 | 2023-08-10T06:07:01.903Z | 10 Acked 1009 Write 2 Done Done Done false |
354 | 2023-08-10T06:07:01.906Z | STATES DS:0 DS:1 DS:2 TOTAL |
355 | 2023-08-10T06:07:01.909Z | New 0 0 0 0 |
356 | 2023-08-10T06:07:01.913Z | Sent 0 0 0 0 |
357 | 2023-08-10T06:07:01.916Z | Done 10 10 10 30 |
358 | 2023-08-10T06:07:01.919Z | Skipped 0 0 0 0 |
359 | 2023-08-10T06:07:01.922Z | Error 0 0 0 0 |
360 | 2023-08-10T06:07:01.926Z | Last Flush: 0 0 0 |
361 | 2023-08-10T06:07:01.929Z | Downstairs last five completed: |
362 | 2023-08-10T06:07:01.933Z | Upstairs last five completed: 10 9 8 7 6 5 |
363 | 2023-08-10T06:07:01.937Z | Wrote out file "/var/tmp/test_up/verify_file" |
364 | 2023-08-10T06:07:01.940Z | Repair part 1 passed |
365 | 2023-08-10T06:07:01.943Z | |
366 | 2023-08-10T06:07:01.947Z | Kill the current downstairs |
367 | 2023-08-10T06:07:01.950Z | rm -rf /var/tmp/test_up/8830 |
368 | 2023-08-10T06:07:01.952Z | Now put back the original so we have a mismatch |
369 | 2023-08-10T06:07:01.956Z | mv /var/tmp/test_up/previous /var/tmp/test_up/8830 |
370 | 2023-08-10T06:07:01.959Z | Restart downstairs with old directory |
371 | 2023-08-10T06:07:02.028Z | /var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 |
372 | 2023-08-10T06:07:02.048Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.159790906Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1200} |
373 | 2023-08-10T06:07:02.052Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.160165308Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1200} |
374 | 2023-08-10T06:07:02.055Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.160207282Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1200} |
375 | 2023-08-10T06:07:02.060Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.17261407Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1200} |
376 | 2023-08-10T06:07:02.063Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.172688803Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1200} |
377 | 2023-08-10T06:07:02.066Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.172714062Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1200} |
378 | 2023-08-10T06:07:02.069Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.179851774Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1200} |
379 | 2023-08-10T06:07:02.072Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.179908629Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1200} |
380 | 2023-08-10T06:07:02.075Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.179951997Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1200} |
381 | 2023-08-10T06:07:02.104Z | EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF |
382 | 2023-08-10T06:07:02.108Z | 0 000-009 26 26 26 2017 2017 2017 F F F |
383 | 2023-08-10T06:07:02.110Z | 1 010-019 26 26 26 2017 2017 2017 T T F <--- |
384 | 2023-08-10T06:07:02.114Z | 2 020-029 26 26 26 2017 2017 2017 T T F <--- |
385 | 2023-08-10T06:07:02.118Z | 3 030-039 26 26 26 2017 2017 2017 T T F <--- |
386 | 2023-08-10T06:07:02.122Z | 4 040-049 26 26 26 2017 2017 2017 T T F <--- |
387 | 2023-08-10T06:07:02.128Z | Max gen: 26, Max flush: 2017 |
388 | 2023-08-10T06:07:02.132Z | Error: Difference in extent metadata found! |
389 | 2023-08-10T06:07:02.135Z | dump test found error as expected |
390 | 2023-08-10T06:07:02.138Z | |
391 | 2023-08-10T06:07:02.141Z | |
392 | 2023-08-10T06:07:02.144Z | /var/tmp/bins/crutest deactivate -g 28 -q --key uUNQZMLN6+7Z1bxrX+G3ctqq/MPXkUp3G36OWVCYmCQ= -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 --verify-out /var/tmp/test_up/verify_file --verify-in /var/tmp/test_up/verify_file |
393 | 2023-08-10T06:07:02.147Z | {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.24599064Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
394 | 2023-08-10T06:07:02.150Z | {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f9f88bb1717d2b86fef41798471b705ff437b87\",\n git_commit_timestamp: \"2023-08-10T05:35:49.000000000Z\",\n git_branch: \"alan/wip-for-vcr\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.246537799Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
395 | 2023-08-10T06:07:02.154Z | {"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.246591049Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
396 | 2023-08-10T06:07:02.157Z | {"msg":"Crucible stats registered with UUID: 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.246634927Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
397 | 2023-08-10T06:07:02.160Z | {"msg":"Crucible 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 has session id: a3136cb6-7313-40df-b89f-a8fe15fe4d2e","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.246678675Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
398 | 2023-08-10T06:07:02.163Z | Crucible runtime is spawned |
399 | 2023-08-10T06:07:02.167Z | The guest has requested activation with gen:28 |
400 | 2023-08-10T06:07:02.170Z | {"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.247029957Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"task":"up_listen"} |
401 | 2023-08-10T06:07:02.174Z | {"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.24709554Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
402 | 2023-08-10T06:07:02.177Z | {"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.24718079Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"looper":"1"} |
403 | 2023-08-10T06:07:02.180Z | {"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.247255562Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
404 | 2023-08-10T06:07:02.183Z | {"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.247345867Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"looper":"0"} |
405 | 2023-08-10T06:07:02.186Z | {"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.247415191Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"looper":"2"} |
406 | 2023-08-10T06:07:02.190Z | {"msg":"[1] 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 looper connected","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.247501989Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"looper":"1"} |
407 | 2023-08-10T06:07:02.193Z | {"msg":"[0] 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 looper connected","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.247557571Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"looper":"0"} |
408 | 2023-08-10T06:07:02.196Z | {"msg":"6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 active request set","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.248086049Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
409 | 2023-08-10T06:07:02.199Z | {"msg":"Set desired generation to :28","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.248161686Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
410 | 2023-08-10T06:07:02.202Z | {"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.248353042Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
411 | 2023-08-10T06:07:02.205Z | {"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.248647692Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
412 | 2023-08-10T06:07:02.208Z | {"msg":"[1] 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 (8659a9be-e58c-4b73-8575-5390f69a5fbc) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.249103869Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
413 | 2023-08-10T06:07:02.211Z | {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.249187161Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
414 | 2023-08-10T06:07:02.215Z | {"msg":"[1] client is_active_req TRUE, promote! session 8659a9be-e58c-4b73-8575-5390f69a5fbc","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.249232068Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
415 | 2023-08-10T06:07:02.219Z | {"msg":"[0] 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 (8659a9be-e58c-4b73-8575-5390f69a5fbc) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.249276844Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
416 | 2023-08-10T06:07:02.222Z | {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.249302496Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
417 | 2023-08-10T06:07:02.225Z | {"msg":"[0] client is_active_req TRUE, promote! session 8659a9be-e58c-4b73-8575-5390f69a5fbc","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.249368463Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
418 | 2023-08-10T06:07:02.227Z | {"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.249789391Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
419 | 2023-08-10T06:07:02.230Z | {"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008820, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.249870188Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
420 | 2023-08-10T06:07:02.233Z | {"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.249943534Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
421 | 2023-08-10T06:07:02.236Z | {"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008810, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.250009359Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
422 | 2023-08-10T06:07:02.239Z | {"msg":"6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.250064319Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
423 | 2023-08-10T06:07:02.242Z | {"msg":"6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.25012241Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
424 | 2023-08-10T06:07:02.245Z | {"msg":"[1] 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 (8659a9be-e58c-4b73-8575-5390f69a5fbc) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.250708139Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
425 | 2023-08-10T06:07:02.249Z | {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.250768881Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
426 | 2023-08-10T06:07:02.252Z | {"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-08-10T06:07:02.250793998Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
427 | 2023-08-10T06:07:02.255Z | {"msg":"[0] 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 (8659a9be-e58c-4b73-8575-5390f69a5fbc) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.250836911Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
428 | 2023-08-10T06:07:02.258Z | {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.250860493Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
429 | 2023-08-10T06:07:02.261Z | {"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-08-10T06:07:02.250899009Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
430 | 2023-08-10T06:07:02.264Z | {"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.250920162Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
431 | 2023-08-10T06:07:02.267Z | {"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.250959083Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
432 | 2023-08-10T06:07:02.270Z | {"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.250981471Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
433 | 2023-08-10T06:07:02.273Z | {"msg":"6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.251022583Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
434 | 2023-08-10T06:07:02.278Z | {"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.251045684Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
435 | 2023-08-10T06:07:02.281Z | {"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.251086445Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
436 | 2023-08-10T06:07:02.284Z | {"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.251107293Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
437 | 2023-08-10T06:07:02.287Z | {"msg":"6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.251151526Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
438 | 2023-08-10T06:07:02.290Z | {"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.251173971Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
439 | 2023-08-10T06:07:02.293Z | {"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:02.251211967Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
440 | 2023-08-10T06:07:03.140Z | {"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:03.252274878Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
441 | 2023-08-10T06:07:04.137Z | {"msg":"[2] 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 looper connected","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.25026458Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"looper":"2"} |
442 | 2023-08-10T06:07:04.140Z | {"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.250364809Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
443 | 2023-08-10T06:07:04.143Z | {"msg":"[2] 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 (8659a9be-e58c-4b73-8575-5390f69a5fbc) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.251235745Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
444 | 2023-08-10T06:07:04.146Z | {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.251296786Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
445 | 2023-08-10T06:07:04.149Z | {"msg":"[2] client is_active_req TRUE, promote! session 8659a9be-e58c-4b73-8575-5390f69a5fbc","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.251342536Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
446 | 2023-08-10T06:07:04.152Z | {"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.251881903Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
447 | 2023-08-10T06:07:04.155Z | {"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 10, shift: 12 }, extent_count: 5, uuid: 12345678-0000-0000-0000-000000008830, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.251942988Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
448 | 2023-08-10T06:07:04.158Z | {"msg":"6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.251987308Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
449 | 2023-08-10T06:07:04.160Z | {"msg":"[2] 6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 (8659a9be-e58c-4b73-8575-5390f69a5fbc) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.252830072Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
450 | 2023-08-10T06:07:04.163Z | {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.25288113Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
451 | 2023-08-10T06:07:04.166Z | {"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-08-10T06:07:04.25297418Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
452 | 2023-08-10T06:07:04.169Z | {"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.25303519Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
453 | 2023-08-10T06:07:04.171Z | {"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253092827Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
454 | 2023-08-10T06:07:04.174Z | {"msg":"6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253132744Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
455 | 2023-08-10T06:07:04.177Z | {"msg":"[1]R flush_numbers: [2017, 2017, 2017, 2017, 2017]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253180309Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
456 | 2023-08-10T06:07:04.180Z | {"msg":"[1]R generation: [26, 26, 26, 26, 26]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253221724Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
457 | 2023-08-10T06:07:04.183Z | {"msg":"[1]R dirty: [false, true, true, true, true]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253262233Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
458 | 2023-08-10T06:07:04.186Z | {"msg":"[0]R flush_numbers: [2017, 2017, 2017, 2017, 2017]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253302302Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
459 | 2023-08-10T06:07:04.189Z | {"msg":"[0]R generation: [26, 26, 26, 26, 26]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253342468Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
460 | 2023-08-10T06:07:04.192Z | {"msg":"[0]R dirty: [false, true, true, true, true]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253382485Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
461 | 2023-08-10T06:07:04.194Z | {"msg":"[2]R flush_numbers: [2017, 2017, 2017, 2017, 2017]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.25340396Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
462 | 2023-08-10T06:07:04.197Z | {"msg":"[2]R generation: [26, 26, 26, 26, 26]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253442263Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
463 | 2023-08-10T06:07:04.200Z | {"msg":"[2]R dirty: [false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253482922Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
464 | 2023-08-10T06:07:04.203Z | {"msg":"Max found gen is 27","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253522036Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
465 | 2023-08-10T06:07:04.205Z | {"msg":"Generation requested: 28 >= found:27","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253561785Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
466 | 2023-08-10T06:07:04.209Z | {"msg":"Next flush: 2018","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253600742Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
467 | 2023-08-10T06:07:04.212Z | {"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253640174Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"mend"} |
468 | 2023-08-10T06:07:04.214Z | {"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253681065Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
469 | 2023-08-10T06:07:04.217Z | {"msg":"extent:1 gens: 26 26 26","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253723467Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
470 | 2023-08-10T06:07:04.220Z | {"msg":"extent:1 flush: 2017 2017 2017 scs: [0, 1, 2]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253765686Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
471 | 2023-08-10T06:07:04.222Z | {"msg":"max_flush now has: [0, 1, 2]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253806982Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
472 | 2023-08-10T06:07:04.226Z | {"msg":"extent:1 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253829335Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
473 | 2023-08-10T06:07:04.228Z | {"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.25389217Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
474 | 2023-08-10T06:07:04.231Z | {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253942825Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
475 | 2023-08-10T06:07:04.233Z | {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.253986491Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
476 | 2023-08-10T06:07:04.236Z | {"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254043673Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"mend"} |
477 | 2023-08-10T06:07:04.239Z | {"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254114593Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
478 | 2023-08-10T06:07:04.242Z | {"msg":"extent:2 gens: 26 26 26","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254193332Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
479 | 2023-08-10T06:07:04.245Z | {"msg":"extent:2 flush: 2017 2017 2017 scs: [0, 1, 2]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254262476Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
480 | 2023-08-10T06:07:04.248Z | {"msg":"max_flush now has: [0, 1, 2]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254341248Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
481 | 2023-08-10T06:07:04.251Z | {"msg":"extent:2 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254403268Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
482 | 2023-08-10T06:07:04.254Z | {"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254461856Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
483 | 2023-08-10T06:07:04.257Z | {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254519248Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
484 | 2023-08-10T06:07:04.260Z | {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254576267Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
485 | 2023-08-10T06:07:04.263Z | {"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.25463517Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"mend"} |
486 | 2023-08-10T06:07:04.266Z | {"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254691516Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
487 | 2023-08-10T06:07:04.268Z | {"msg":"extent:3 gens: 26 26 26","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254747957Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
488 | 2023-08-10T06:07:04.271Z | {"msg":"extent:3 flush: 2017 2017 2017 scs: [0, 1, 2]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254807088Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
489 | 2023-08-10T06:07:04.274Z | {"msg":"max_flush now has: [0, 1, 2]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254863863Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
490 | 2023-08-10T06:07:04.277Z | {"msg":"extent:3 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254920243Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
491 | 2023-08-10T06:07:04.280Z | {"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.254977039Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
492 | 2023-08-10T06:07:04.283Z | {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.25503339Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
493 | 2023-08-10T06:07:04.286Z | {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255090972Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
494 | 2023-08-10T06:07:04.289Z | {"msg":"Extents 4 dirty","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255147958Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"mend"} |
495 | 2023-08-10T06:07:04.292Z | {"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255208997Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
496 | 2023-08-10T06:07:04.295Z | {"msg":"extent:4 gens: 26 26 26","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255278539Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
497 | 2023-08-10T06:07:04.297Z | {"msg":"extent:4 flush: 2017 2017 2017 scs: [0, 1, 2]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255340515Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
498 | 2023-08-10T06:07:04.300Z | {"msg":"max_flush now has: [0, 1, 2]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255397517Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
499 | 2023-08-10T06:07:04.303Z | {"msg":"extent:4 dirty: true true false","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255454082Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
500 | 2023-08-10T06:07:04.306Z | {"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255510429Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
501 | 2023-08-10T06:07:04.309Z | {"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255566814Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
502 | 2023-08-10T06:07:04.312Z | {"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255631229Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"mrl":"dirty","":"mend"} |
503 | 2023-08-10T06:07:04.315Z | {"msg":"Transition from WaitQuorum to Repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255689325Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
504 | 2023-08-10T06:07:04.318Z | {"msg":"Transition from WaitQuorum to Repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255744398Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
505 | 2023-08-10T06:07:04.320Z | {"msg":"Transition from WaitQuorum to Repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255799959Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
506 | 2023-08-10T06:07:04.323Z | {"msg":"Found 4 extents that need repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255855085Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
507 | 2023-08-10T06:07:04.326Z | {"msg":"Full repair list: {4: ExtentFix { source: 0, dest: [1, 2] }, 3: ExtentFix { source: 0, dest: [1, 2] }, 1: ExtentFix { source: 0, dest: [1, 2] }, 2: ExtentFix { source: 0, dest: [1, 2] }}","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.255916802Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
508 | 2023-08-10T06:07:04.329Z | {"msg":"Task list: [ReconcileIO { id: 0, op: ExtentFlush { repair_id: 0, extent_id: 4, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {1: New, 2: New, 0: New} }, ReconcileIO { id: 1, op: ExtentClose { repair_id: 1, extent_id: 4 }, state: {0: New, 1: New, 2: New} }, ReconcileIO { id: 2, op: ExtentRepair { repair_id: 2, extent_id: 4, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {2: New, 0: New, 1: New} }, ReconcileIO { id: 3, op: ExtentReopen { repair_id: 3, extent_id: 4 }, state: {1: New, 0: New, 2: New} }, ReconcileIO { id: 4, op: ExtentFlush { repair_id: 4, extent_id: 3, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {0: New, 2: New, 1: New} }, ReconcileIO { id: 5, op: ExtentClose { repair_id: 5, extent_id: 3 }, state: {1: New, 0: New, 2: New} }, ReconcileIO { id: 6, op: ExtentRepair { repair_id: 6, extent_id: 3, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 0: New, 2: New} }, ReconcileIO { id: 7, op: ExtentReopen { repair_id: 7, extent_id: 3 }, state: {1: New, 2: New, 0: New} }, ReconcileIO { id: 8, op: ExtentFlush { repair_id: 8, extent_id: 1, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {2: New, 1: New, 0: New} }, ReconcileIO { id: 9, op: ExtentClose { repair_id: 9, extent_id: 1 }, state: {0: New, 1: New, 2: New} }, ReconcileIO { id: 10, op: ExtentRepair { repair_id: 10, extent_id: 1, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 2: New, 0: New} }, ReconcileIO { id: 11, op: ExtentReopen { repair_id: 11, extent_id: 1 }, state: {1: New, 0: New, 2: New} }, ReconcileIO { id: 12, op: ExtentFlush { repair_id: 12, extent_id: 2, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {0: New, 1: New, 2: New} }, ReconcileIO { id: 13, op: ExtentClose { repair_id: 13, extent_id: 2 }, state: {1: New, 2: New, 0: New} }, ReconcileIO { id: 14, op: ExtentRepair { repair_id: 14, extent_id: 2, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 0: New, 2: New} }, ReconcileIO { id: 15, op: ExtentReopen { repair_id: 15, extent_id: 2 }, state: {0: New, 1: New, 2: New} }]","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256051507Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
509 | 2023-08-10T06:07:04.332Z | {"msg":"Begin repair with 16 commands","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256081545Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
510 | 2023-08-10T06:07:04.336Z | {"msg":"Pop front: ReconcileIO { id: 0, op: ExtentFlush { repair_id: 0, extent_id: 4, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {1: New, 2: New, 0: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256109331Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
511 | 2023-08-10T06:07:04.338Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256139459Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
512 | 2023-08-10T06:07:04.341Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256159981Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
513 | 2023-08-10T06:07:04.344Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 0, op: ExtentFlush { repair_id: 0, extent_id: 4, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {1: New, 2: InProgress, 0: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256182478Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
514 | 2023-08-10T06:07:04.347Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.25620262Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
515 | 2023-08-10T06:07:04.349Z | {"msg":"[2] client ExtentFlush { repair_id: 0, extent_id: 4, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256223288Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
516 | 2023-08-10T06:07:04.352Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 0, op: ExtentFlush { repair_id: 0, extent_id: 4, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {1: New, 2: InProgress, 0: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256245829Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
517 | 2023-08-10T06:07:04.355Z | {"msg":"[0] client ExtentFlush { repair_id: 0, extent_id: 4, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256267754Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
518 | 2023-08-10T06:07:04.358Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256286201Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
519 | 2023-08-10T06:07:04.361Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 0, op: ExtentFlush { repair_id: 0, extent_id: 4, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {1: InProgress, 2: Done, 0: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256309461Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
520 | 2023-08-10T06:07:04.364Z | {"msg":"[1] client ExtentFlush { repair_id: 0, extent_id: 4, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.256331871Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
521 | 2023-08-10T06:07:04.367Z | {"msg":"Completion from [0] id:0 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.25968067Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
522 | 2023-08-10T06:07:04.370Z | {"msg":"[1/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.259713019Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
523 | 2023-08-10T06:07:04.373Z | {"msg":"Pop front: ReconcileIO { id: 1, op: ExtentClose { repair_id: 1, extent_id: 4 }, state: {0: New, 1: New, 2: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.2597354Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
524 | 2023-08-10T06:07:04.377Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.259755099Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
525 | 2023-08-10T06:07:04.381Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.259773865Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
526 | 2023-08-10T06:07:04.384Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 1, op: ExtentClose { repair_id: 1, extent_id: 4 }, state: {0: New, 1: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.259795339Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
527 | 2023-08-10T06:07:04.387Z | {"msg":"[2] client ExtentClose { repair_id: 1, extent_id: 4 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.259817259Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
528 | 2023-08-10T06:07:04.390Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.259836215Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
529 | 2023-08-10T06:07:04.393Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 1, op: ExtentClose { repair_id: 1, extent_id: 4 }, state: {0: InProgress, 1: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.259857388Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
530 | 2023-08-10T06:07:04.396Z | {"msg":"[0] client ExtentClose { repair_id: 1, extent_id: 4 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.259877916Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
531 | 2023-08-10T06:07:04.399Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.259916046Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
532 | 2023-08-10T06:07:04.401Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 1, op: ExtentClose { repair_id: 1, extent_id: 4 }, state: {0: InProgress, 1: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.25995176Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
533 | 2023-08-10T06:07:04.404Z | {"msg":"[1] client ExtentClose { repair_id: 1, extent_id: 4 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.259975098Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
534 | 2023-08-10T06:07:04.407Z | {"msg":"Completion from [1] id:1 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263121779Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
535 | 2023-08-10T06:07:04.410Z | {"msg":"[2/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263161732Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
536 | 2023-08-10T06:07:04.413Z | {"msg":"Pop front: ReconcileIO { id: 2, op: ExtentRepair { repair_id: 2, extent_id: 4, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {2: New, 0: New, 1: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263187885Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
537 | 2023-08-10T06:07:04.417Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263208264Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
538 | 2023-08-10T06:07:04.420Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263227652Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
539 | 2023-08-10T06:07:04.423Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 2, op: ExtentRepair { repair_id: 2, extent_id: 4, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {2: InProgress, 0: New, 1: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263251391Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
540 | 2023-08-10T06:07:04.426Z | {"msg":"[2] client ExtentRepair { repair_id: 2, extent_id: 4, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263274655Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
541 | 2023-08-10T06:07:04.430Z | {"msg":"[2] Sending repair request 2","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263294016Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
542 | 2023-08-10T06:07:04.432Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263312528Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
543 | 2023-08-10T06:07:04.435Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 2, op: ExtentRepair { repair_id: 2, extent_id: 4, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {2: InProgress, 0: InProgress, 1: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263336248Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
544 | 2023-08-10T06:07:04.439Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263356244Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
545 | 2023-08-10T06:07:04.442Z | {"msg":"[0] client ExtentRepair { repair_id: 2, extent_id: 4, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263377835Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
546 | 2023-08-10T06:07:04.445Z | {"msg":"[0] No action required 2","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263396612Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
547 | 2023-08-10T06:07:04.448Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 2, op: ExtentRepair { repair_id: 2, extent_id: 4, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {2: InProgress, 0: InProgress, 1: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263420036Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
548 | 2023-08-10T06:07:04.451Z | {"msg":"[1] client ExtentRepair { repair_id: 2, extent_id: 4, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.263442645Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
549 | 2023-08-10T06:07:04.454Z | {"msg":"[1] Sending repair request 2","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.26361193Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
550 | 2023-08-10T06:07:04.456Z | {"msg":"Completion from [1] id:2 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409048026Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
551 | 2023-08-10T06:07:04.459Z | {"msg":"[3/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409154835Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
552 | 2023-08-10T06:07:04.462Z | {"msg":"Pop front: ReconcileIO { id: 3, op: ExtentReopen { repair_id: 3, extent_id: 4 }, state: {1: New, 0: New, 2: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409189435Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
553 | 2023-08-10T06:07:04.465Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.40925248Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
554 | 2023-08-10T06:07:04.468Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409317192Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
555 | 2023-08-10T06:07:04.471Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 3, op: ExtentReopen { repair_id: 3, extent_id: 4 }, state: {1: New, 0: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409393766Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
556 | 2023-08-10T06:07:04.474Z | {"msg":"[2] client ExtentReopen { repair_id: 3, extent_id: 4 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409473432Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
557 | 2023-08-10T06:07:04.477Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409513648Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
558 | 2023-08-10T06:07:04.479Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 3, op: ExtentReopen { repair_id: 3, extent_id: 4 }, state: {1: New, 0: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409585101Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
559 | 2023-08-10T06:07:04.483Z | {"msg":"[0] client ExtentReopen { repair_id: 3, extent_id: 4 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409656069Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
560 | 2023-08-10T06:07:04.486Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.40972659Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
561 | 2023-08-10T06:07:04.489Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 3, op: ExtentReopen { repair_id: 3, extent_id: 4 }, state: {1: InProgress, 0: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409772345Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
562 | 2023-08-10T06:07:04.492Z | {"msg":"[1] client ExtentReopen { repair_id: 3, extent_id: 4 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.409837142Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
563 | 2023-08-10T06:07:04.494Z | {"msg":"Completion from [1] id:3 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410470732Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
564 | 2023-08-10T06:07:04.497Z | {"msg":"[4/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410513157Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
565 | 2023-08-10T06:07:04.500Z | {"msg":"Pop front: ReconcileIO { id: 4, op: ExtentFlush { repair_id: 4, extent_id: 3, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {0: New, 2: New, 1: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410555547Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
566 | 2023-08-10T06:07:04.503Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410592564Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
567 | 2023-08-10T06:07:04.506Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410630214Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
568 | 2023-08-10T06:07:04.509Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 4, op: ExtentFlush { repair_id: 4, extent_id: 3, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {0: New, 2: InProgress, 1: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410673985Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
569 | 2023-08-10T06:07:04.513Z | {"msg":"[2] client ExtentFlush { repair_id: 4, extent_id: 3, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.41071811Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
570 | 2023-08-10T06:07:04.516Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410744334Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
571 | 2023-08-10T06:07:04.518Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 4, op: ExtentFlush { repair_id: 4, extent_id: 3, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {0: New, 2: Done, 1: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410767896Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
572 | 2023-08-10T06:07:04.521Z | {"msg":"[1] client ExtentFlush { repair_id: 4, extent_id: 3, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.41079075Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
573 | 2023-08-10T06:07:04.524Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410809815Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
574 | 2023-08-10T06:07:04.527Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 4, op: ExtentFlush { repair_id: 4, extent_id: 3, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {0: InProgress, 2: Done, 1: Done} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410832714Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
575 | 2023-08-10T06:07:04.534Z | {"msg":"[0] client ExtentFlush { repair_id: 4, extent_id: 3, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.410854448Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
576 | 2023-08-10T06:07:04.537Z | {"msg":"Completion from [0] id:4 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.415938605Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
577 | 2023-08-10T06:07:04.540Z | {"msg":"[5/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.415971406Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
578 | 2023-08-10T06:07:04.542Z | {"msg":"Pop front: ReconcileIO { id: 5, op: ExtentClose { repair_id: 5, extent_id: 3 }, state: {1: New, 0: New, 2: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.415993406Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
579 | 2023-08-10T06:07:04.545Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.416013139Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
580 | 2023-08-10T06:07:04.548Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.416039435Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
581 | 2023-08-10T06:07:04.551Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 5, op: ExtentClose { repair_id: 5, extent_id: 3 }, state: {1: New, 0: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.416079943Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
582 | 2023-08-10T06:07:04.554Z | {"msg":"[2] client ExtentClose { repair_id: 5, extent_id: 3 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.416119448Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
583 | 2023-08-10T06:07:04.557Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.416158408Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
584 | 2023-08-10T06:07:04.560Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 5, op: ExtentClose { repair_id: 5, extent_id: 3 }, state: {1: New, 0: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.416200056Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
585 | 2023-08-10T06:07:04.563Z | {"msg":"[0] client ExtentClose { repair_id: 5, extent_id: 3 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.416241302Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
586 | 2023-08-10T06:07:04.566Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.416264543Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
587 | 2023-08-10T06:07:04.569Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 5, op: ExtentClose { repair_id: 5, extent_id: 3 }, state: {1: InProgress, 0: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.416287163Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
588 | 2023-08-10T06:07:04.572Z | {"msg":"[1] client ExtentClose { repair_id: 5, extent_id: 3 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.416307886Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
589 | 2023-08-10T06:07:04.575Z | {"msg":"Completion from [1] id:5 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419279579Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
590 | 2023-08-10T06:07:04.580Z | {"msg":"[6/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419310345Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
591 | 2023-08-10T06:07:04.583Z | {"msg":"Pop front: ReconcileIO { id: 6, op: ExtentRepair { repair_id: 6, extent_id: 3, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 0: New, 2: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419334263Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
592 | 2023-08-10T06:07:04.586Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419353965Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
593 | 2023-08-10T06:07:04.589Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419373603Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
594 | 2023-08-10T06:07:04.592Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 6, op: ExtentRepair { repair_id: 6, extent_id: 3, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 0: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419397135Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
595 | 2023-08-10T06:07:04.595Z | {"msg":"[2] client ExtentRepair { repair_id: 6, extent_id: 3, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419420478Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
596 | 2023-08-10T06:07:04.598Z | {"msg":"[2] Sending repair request 6","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419444265Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
597 | 2023-08-10T06:07:04.601Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419477732Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
598 | 2023-08-10T06:07:04.603Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 6, op: ExtentRepair { repair_id: 6, extent_id: 3, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 0: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419508225Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
599 | 2023-08-10T06:07:04.606Z | {"msg":"[0] client ExtentRepair { repair_id: 6, extent_id: 3, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.41953139Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
600 | 2023-08-10T06:07:04.609Z | {"msg":"[0] No action required 6","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419550582Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
601 | 2023-08-10T06:07:04.612Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419569277Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
602 | 2023-08-10T06:07:04.615Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 6, op: ExtentRepair { repair_id: 6, extent_id: 3, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: InProgress, 0: Done, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419592889Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
603 | 2023-08-10T06:07:04.618Z | {"msg":"[1] client ExtentRepair { repair_id: 6, extent_id: 3, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419615139Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
604 | 2023-08-10T06:07:04.621Z | {"msg":"[1] Sending repair request 6","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.419634353Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
605 | 2023-08-10T06:07:04.624Z | {"msg":"Completion from [1] id:6 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500284559Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
606 | 2023-08-10T06:07:04.627Z | {"msg":"[7/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500338709Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
607 | 2023-08-10T06:07:04.630Z | {"msg":"Pop front: ReconcileIO { id: 7, op: ExtentReopen { repair_id: 7, extent_id: 3 }, state: {1: New, 2: New, 0: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500361403Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
608 | 2023-08-10T06:07:04.633Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500381765Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
609 | 2023-08-10T06:07:04.637Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500400792Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
610 | 2023-08-10T06:07:04.640Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 7, op: ExtentReopen { repair_id: 7, extent_id: 3 }, state: {1: New, 2: InProgress, 0: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500422444Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
611 | 2023-08-10T06:07:04.644Z | {"msg":"[2] client ExtentReopen { repair_id: 7, extent_id: 3 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500455952Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
612 | 2023-08-10T06:07:04.646Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500488526Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
613 | 2023-08-10T06:07:04.650Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 7, op: ExtentReopen { repair_id: 7, extent_id: 3 }, state: {1: New, 2: InProgress, 0: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500513195Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
614 | 2023-08-10T06:07:04.652Z | {"msg":"[0] client ExtentReopen { repair_id: 7, extent_id: 3 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500534759Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
615 | 2023-08-10T06:07:04.655Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500553621Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
616 | 2023-08-10T06:07:04.658Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 7, op: ExtentReopen { repair_id: 7, extent_id: 3 }, state: {1: InProgress, 2: InProgress, 0: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500574995Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
617 | 2023-08-10T06:07:04.661Z | {"msg":"[1] client ExtentReopen { repair_id: 7, extent_id: 3 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.500595329Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
618 | 2023-08-10T06:07:04.663Z | {"msg":"Completion from [1] id:7 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501586351Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
619 | 2023-08-10T06:07:04.666Z | {"msg":"[8/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501614738Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
620 | 2023-08-10T06:07:04.669Z | {"msg":"Pop front: ReconcileIO { id: 8, op: ExtentFlush { repair_id: 8, extent_id: 1, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {2: New, 1: New, 0: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501637296Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
621 | 2023-08-10T06:07:04.672Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501727144Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
622 | 2023-08-10T06:07:04.674Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501768306Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
623 | 2023-08-10T06:07:04.677Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 8, op: ExtentFlush { repair_id: 8, extent_id: 1, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {2: InProgress, 1: New, 0: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501811593Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
624 | 2023-08-10T06:07:04.680Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501849317Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
625 | 2023-08-10T06:07:04.683Z | {"msg":"[2] client ExtentFlush { repair_id: 8, extent_id: 1, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501888587Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
626 | 2023-08-10T06:07:04.686Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 8, op: ExtentFlush { repair_id: 8, extent_id: 1, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {2: Done, 1: New, 0: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501928455Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
627 | 2023-08-10T06:07:04.688Z | {"msg":"[0] client ExtentFlush { repair_id: 8, extent_id: 1, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501953367Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
628 | 2023-08-10T06:07:04.692Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501972576Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
629 | 2023-08-10T06:07:04.695Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 8, op: ExtentFlush { repair_id: 8, extent_id: 1, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {2: Done, 1: InProgress, 0: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.501995602Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
630 | 2023-08-10T06:07:04.698Z | {"msg":"[1] client ExtentFlush { repair_id: 8, extent_id: 1, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.502017134Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
631 | 2023-08-10T06:07:04.701Z | {"msg":"Completion from [0] id:8 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506670344Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
632 | 2023-08-10T06:07:04.704Z | {"msg":"[9/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506700083Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
633 | 2023-08-10T06:07:04.706Z | {"msg":"Pop front: ReconcileIO { id: 9, op: ExtentClose { repair_id: 9, extent_id: 1 }, state: {0: New, 1: New, 2: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.50672192Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
634 | 2023-08-10T06:07:04.709Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506741936Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
635 | 2023-08-10T06:07:04.712Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506761346Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
636 | 2023-08-10T06:07:04.715Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 9, op: ExtentClose { repair_id: 9, extent_id: 1 }, state: {0: New, 1: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506783319Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
637 | 2023-08-10T06:07:04.718Z | {"msg":"[2] client ExtentClose { repair_id: 9, extent_id: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506809135Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
638 | 2023-08-10T06:07:04.720Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506842833Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
639 | 2023-08-10T06:07:04.723Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 9, op: ExtentClose { repair_id: 9, extent_id: 1 }, state: {0: InProgress, 1: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506871613Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
640 | 2023-08-10T06:07:04.726Z | {"msg":"[0] client ExtentClose { repair_id: 9, extent_id: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506893372Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
641 | 2023-08-10T06:07:04.729Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506912909Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
642 | 2023-08-10T06:07:04.732Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 9, op: ExtentClose { repair_id: 9, extent_id: 1 }, state: {0: InProgress, 1: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506935517Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
643 | 2023-08-10T06:07:04.735Z | {"msg":"[1] client ExtentClose { repair_id: 9, extent_id: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.506956061Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
644 | 2023-08-10T06:07:04.738Z | {"msg":"Completion from [1] id:9 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.509812035Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
645 | 2023-08-10T06:07:04.740Z | {"msg":"[10/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.509843732Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
646 | 2023-08-10T06:07:04.743Z | {"msg":"Pop front: ReconcileIO { id: 10, op: ExtentRepair { repair_id: 10, extent_id: 1, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 2: New, 0: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.50987888Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
647 | 2023-08-10T06:07:04.746Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.509916746Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
648 | 2023-08-10T06:07:04.749Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.509952364Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
649 | 2023-08-10T06:07:04.751Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 10, op: ExtentRepair { repair_id: 10, extent_id: 1, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 2: InProgress, 0: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.509997626Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
650 | 2023-08-10T06:07:04.754Z | {"msg":"[2] client ExtentRepair { repair_id: 10, extent_id: 1, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.510041666Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
651 | 2023-08-10T06:07:04.757Z | {"msg":"[2] Sending repair request 10","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.510079575Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
652 | 2023-08-10T06:07:04.760Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.510116233Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
653 | 2023-08-10T06:07:04.762Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.510153412Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
654 | 2023-08-10T06:07:04.765Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 10, op: ExtentRepair { repair_id: 10, extent_id: 1, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 2: InProgress, 0: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.510196867Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
655 | 2023-08-10T06:07:04.768Z | {"msg":"[0] client ExtentRepair { repair_id: 10, extent_id: 1, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.510224209Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
656 | 2023-08-10T06:07:04.770Z | {"msg":"[0] No action required 10","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.510243354Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
657 | 2023-08-10T06:07:04.773Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 10, op: ExtentRepair { repair_id: 10, extent_id: 1, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: InProgress, 2: InProgress, 0: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.510267444Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
658 | 2023-08-10T06:07:04.776Z | {"msg":"[1] client ExtentRepair { repair_id: 10, extent_id: 1, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.510290752Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
659 | 2023-08-10T06:07:04.779Z | {"msg":"[1] Sending repair request 10","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.510310002Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
660 | 2023-08-10T06:07:04.781Z | {"msg":"Completion from [2] id:10 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.591326268Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
661 | 2023-08-10T06:07:04.784Z | {"msg":"[11/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.591395145Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
662 | 2023-08-10T06:07:04.787Z | {"msg":"Pop front: ReconcileIO { id: 11, op: ExtentReopen { repair_id: 11, extent_id: 1 }, state: {1: New, 0: New, 2: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.591418777Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
663 | 2023-08-10T06:07:04.790Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.591438831Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
664 | 2023-08-10T06:07:04.793Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.59145811Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
665 | 2023-08-10T06:07:04.795Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 11, op: ExtentReopen { repair_id: 11, extent_id: 1 }, state: {1: New, 0: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.59148094Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
666 | 2023-08-10T06:07:04.798Z | {"msg":"[2] client ExtentReopen { repair_id: 11, extent_id: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.591514166Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
667 | 2023-08-10T06:07:04.801Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.59154903Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
668 | 2023-08-10T06:07:04.804Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 11, op: ExtentReopen { repair_id: 11, extent_id: 1 }, state: {1: New, 0: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.591576069Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
669 | 2023-08-10T06:07:04.806Z | {"msg":"[0] client ExtentReopen { repair_id: 11, extent_id: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.591597393Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
670 | 2023-08-10T06:07:04.809Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.591616151Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
671 | 2023-08-10T06:07:04.812Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 11, op: ExtentReopen { repair_id: 11, extent_id: 1 }, state: {1: InProgress, 0: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.591638792Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
672 | 2023-08-10T06:07:04.815Z | {"msg":"[1] client ExtentReopen { repair_id: 11, extent_id: 1 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.591659824Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
673 | 2023-08-10T06:07:04.817Z | {"msg":"Completion from [1] id:11 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.592827843Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
674 | 2023-08-10T06:07:04.820Z | {"msg":"[12/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.592860604Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
675 | 2023-08-10T06:07:04.823Z | {"msg":"Pop front: ReconcileIO { id: 12, op: ExtentFlush { repair_id: 12, extent_id: 2, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {0: New, 1: New, 2: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.592884576Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
676 | 2023-08-10T06:07:04.826Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.592904371Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
677 | 2023-08-10T06:07:04.829Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.592926804Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
678 | 2023-08-10T06:07:04.831Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 12, op: ExtentFlush { repair_id: 12, extent_id: 2, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {0: New, 1: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.592961727Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
679 | 2023-08-10T06:07:04.834Z | {"msg":"[2] client ExtentFlush { repair_id: 12, extent_id: 2, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.593002378Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
680 | 2023-08-10T06:07:04.837Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.593025563Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
681 | 2023-08-10T06:07:04.839Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.593045Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
682 | 2023-08-10T06:07:04.842Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 12, op: ExtentFlush { repair_id: 12, extent_id: 2, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {0: InProgress, 1: New, 2: Done} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.593068786Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
683 | 2023-08-10T06:07:04.845Z | {"msg":"[0] client ExtentFlush { repair_id: 12, extent_id: 2, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.593090783Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
684 | 2023-08-10T06:07:04.848Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 12, op: ExtentFlush { repair_id: 12, extent_id: 2, client_id: 0, flush_number: 2018, gen_number: 27 }, state: {0: InProgress, 1: InProgress, 2: Done} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.593113809Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
685 | 2023-08-10T06:07:04.851Z | {"msg":"[1] client ExtentFlush { repair_id: 12, extent_id: 2, client_id: 0, flush_number: 2018, gen_number: 27 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.593142124Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
686 | 2023-08-10T06:07:04.853Z | {"msg":"Completion from [0] id:12 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.597929041Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
687 | 2023-08-10T06:07:04.856Z | {"msg":"[13/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.597963494Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
688 | 2023-08-10T06:07:04.859Z | {"msg":"Pop front: ReconcileIO { id: 13, op: ExtentClose { repair_id: 13, extent_id: 2 }, state: {1: New, 2: New, 0: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.597986045Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
689 | 2023-08-10T06:07:04.862Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.598005285Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
690 | 2023-08-10T06:07:04.864Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.598025156Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
691 | 2023-08-10T06:07:04.867Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 13, op: ExtentClose { repair_id: 13, extent_id: 2 }, state: {1: New, 2: InProgress, 0: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.598047089Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
692 | 2023-08-10T06:07:04.870Z | {"msg":"[2] client ExtentClose { repair_id: 13, extent_id: 2 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.598068455Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
693 | 2023-08-10T06:07:04.873Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.59809045Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
694 | 2023-08-10T06:07:04.876Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 13, op: ExtentClose { repair_id: 13, extent_id: 2 }, state: {1: New, 2: InProgress, 0: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.598129664Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
695 | 2023-08-10T06:07:04.878Z | {"msg":"[0] client ExtentClose { repair_id: 13, extent_id: 2 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.598164894Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
696 | 2023-08-10T06:07:04.881Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.598186451Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
697 | 2023-08-10T06:07:04.883Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 13, op: ExtentClose { repair_id: 13, extent_id: 2 }, state: {1: InProgress, 2: InProgress, 0: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.598208888Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
698 | 2023-08-10T06:07:04.886Z | {"msg":"[1] client ExtentClose { repair_id: 13, extent_id: 2 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.59822932Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
699 | 2023-08-10T06:07:04.889Z | {"msg":"Completion from [1] id:13 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.601496636Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
700 | 2023-08-10T06:07:04.892Z | {"msg":"[14/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.601529954Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
701 | 2023-08-10T06:07:04.895Z | {"msg":"Pop front: ReconcileIO { id: 14, op: ExtentRepair { repair_id: 14, extent_id: 2, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 0: New, 2: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.601554954Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
702 | 2023-08-10T06:07:04.897Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.601574944Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
703 | 2023-08-10T06:07:04.900Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.60159469Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
704 | 2023-08-10T06:07:04.903Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 14, op: ExtentRepair { repair_id: 14, extent_id: 2, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 0: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.601619395Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
705 | 2023-08-10T06:07:04.905Z | {"msg":"[2] client ExtentRepair { repair_id: 14, extent_id: 2, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.601651362Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
706 | 2023-08-10T06:07:04.908Z | {"msg":"[2] Sending repair request 14","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.601687474Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
707 | 2023-08-10T06:07:04.911Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.601711121Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
708 | 2023-08-10T06:07:04.914Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 14, op: ExtentRepair { repair_id: 14, extent_id: 2, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: New, 0: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.601735464Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
709 | 2023-08-10T06:07:04.916Z | {"msg":"[0] client ExtentRepair { repair_id: 14, extent_id: 2, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.601769283Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
710 | 2023-08-10T06:07:04.919Z | {"msg":"[0] No action required 14","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.641594164Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
711 | 2023-08-10T06:07:04.922Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.641646998Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
712 | 2023-08-10T06:07:04.924Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 14, op: ExtentRepair { repair_id: 14, extent_id: 2, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }, state: {1: InProgress, 0: Done, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.641691194Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
713 | 2023-08-10T06:07:04.927Z | {"msg":"[1] client ExtentRepair { repair_id: 14, extent_id: 2, source_client_id: 0, source_repair_address: 0.0.0.0:12810, dest_clients: [1, 2] }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.641737384Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
714 | 2023-08-10T06:07:04.930Z | {"msg":"[1] Sending repair request 14","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.641774432Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
715 | 2023-08-10T06:07:04.933Z | {"msg":"Completion from [2] id:14 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685560572Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
716 | 2023-08-10T06:07:04.936Z | {"msg":"[15/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685608172Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
717 | 2023-08-10T06:07:04.938Z | {"msg":"Pop front: ReconcileIO { id: 15, op: ExtentReopen { repair_id: 15, extent_id: 2 }, state: {0: New, 1: New, 2: New} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685632349Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
718 | 2023-08-10T06:07:04.941Z | {"msg":"Sent repair work, now wait for resp","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685652705Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
719 | 2023-08-10T06:07:04.944Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685671859Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
720 | 2023-08-10T06:07:04.946Z | {"msg":"[2] rep_in_progress: return ReconcileIO { id: 15, op: ExtentReopen { repair_id: 15, extent_id: 2 }, state: {0: New, 1: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685694053Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
721 | 2023-08-10T06:07:04.949Z | {"msg":"[2] client ExtentReopen { repair_id: 15, extent_id: 2 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685715596Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
722 | 2023-08-10T06:07:04.952Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.68573462Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
723 | 2023-08-10T06:07:04.955Z | {"msg":"[0] rep_in_progress: return ReconcileIO { id: 15, op: ExtentReopen { repair_id: 15, extent_id: 2 }, state: {0: InProgress, 1: New, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685757541Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
724 | 2023-08-10T06:07:04.957Z | {"msg":"[0] client ExtentReopen { repair_id: 15, extent_id: 2 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685784587Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
725 | 2023-08-10T06:07:04.960Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685819534Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
726 | 2023-08-10T06:07:04.963Z | {"msg":"[1] rep_in_progress: return ReconcileIO { id: 15, op: ExtentReopen { repair_id: 15, extent_id: 2 }, state: {0: InProgress, 1: InProgress, 2: InProgress} }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685861907Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201,"":"downstairs"} |
727 | 2023-08-10T06:07:04.974Z | {"msg":"[1] client ExtentReopen { repair_id: 15, extent_id: 2 }","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.685904324Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
728 | 2023-08-10T06:07:04.977Z | {"msg":"Completion from [0] id:15 status:true","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.68699125Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
729 | 2023-08-10T06:07:04.979Z | {"msg":"[16/16] Repair commands completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687028228Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
730 | 2023-08-10T06:07:04.982Z | {"msg":"4 extents repaired in 0.433 ave:0.1082","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687049474Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
731 | 2023-08-10T06:07:04.984Z | {"msg":"All required repair work is completed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687069005Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
732 | 2023-08-10T06:07:04.987Z | {"msg":"Set Downstairs and Upstairs active after repairs","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687088079Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
733 | 2023-08-10T06:07:04.990Z | {"msg":"6aac1c0a-c22f-4cb2-8a7b-6a31d4c53787 is now active with session: 8659a9be-e58c-4b73-8575-5390f69a5fbc","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687109717Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
734 | 2023-08-10T06:07:04.992Z | {"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level"The guest has finished waiting for activation with:28 |
735 | 2023-08-10T06:07:04.995Z | :30Wait for a query_work_queue command to finish before sending IO |
736 | 2023-08-10T06:07:04.998Z | ,"time":"2023-08-10T06:07:04.687135642Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
737 | 2023-08-10T06:07:05.002Z | {"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687170127Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
738 | 2023-08-10T06:07:05.005Z | {"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687316562Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
739 | 2023-08-10T06:07:05.008Z | {"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.68734657Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
740 | 2023-08-10T06:07:05.011Z | {"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687365421Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
741 | 2023-08-10T06:07:05.014Z | {"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687750231Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
742 | 2023-08-10T06:07:05.016Z | {"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.68784782Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
743 | 2023-08-10T06:07:05.019Z | {"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.68788037Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
744 | 2023-08-10T06:07:05.022Z | {"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687899734Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
745 | 2023-08-10T06:07:05.025Z | {"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687918621Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
746 | 2023-08-10T06:07:05.028Z | {"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:04.687936908Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
747 | 2023-08-10T06:07:05.031Z | Region: es:10 ec:5 bs:4096 ts:204800 tb:50 max_io:50 or 204800 |
748 | 2023-08-10T06:07:05.034Z | Loading write count information from file "/var/tmp/test_up/verify_file" |
749 | 2023-08-10T06:07:05.036Z | Read and Verify all blocks (0..50 range:false) |
750 | 2023-08-10T06:07:05.040Z | Wrote out file "/var/tmp/test_up/verify_file" |
751 | 2023-08-10T06:07:05.043Z | Verify test completed |
752 | 2023-08-10T06:07:05.046Z | Wrote out file "/var/tmp/test_up/verify_file" |
753 | 2023-08-10T06:07:05.049Z | CLIENT: Tests done. All submitted work has been ACK'd |
754 | 2023-08-10T06:07:05.052Z | ---------------------------------------------------------------- |
755 | 2023-08-10T06:07:05.055Z | Crucible gen:28 GIO:true work queues: Upstairs:1 downstairs:2 |
756 | 2023-08-10T06:07:05.059Z | GW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY |
757 | 2023-08-10T06:07:05.063Z | 1 Acked 1000 Read 50 Done Done Done false |
758 | 2023-08-10T06:07:05.066Z | 2 NotAcked 1001 Flush 0 Sent Sent Sent false |
759 | 2023-08-10T06:07:05.069Z | STATES DS:0 DS:1 DS:2 TOTAL |
760 | 2023-08-10T06:07:05.072Z | New 0 0 0 0 |
761 | 2023-08-10T06:07:05.077Z | Sent 1 1 1 3 |
762 | 2023-08-10T06:07:05.080Z | Done 1 1 1 3 |
763 | 2023-08-10T06:07:05.083Z | Skipped 0 0 0 0 |
764 | 2023-08-10T06:07:05.087Z | Error 0 0 0 0 |
765 | 2023-08-10T06:07:05.090Z | Last Flush: 0 0 0 |
766 | 2023-08-10T06:07:05.096Z | Downstairs last five completed: |
767 | 2023-08-10T06:07:05.100Z | Upstairs last five completed: 1 |
768 | 2023-08-10T06:07:05.105Z | CLIENT: Up:1 ds:2 act:3 |
769 | 2023-08-10T06:07:05.574Z | {"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:05.688546929Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
770 | 2023-08-10T06:07:05.577Z | {"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:05.68864259Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1201} |
771 | 2023-08-10T06:07:08.827Z | ---------------------------------------------------------------- |
772 | 2023-08-10T06:07:08.830Z | Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:0 |
773 | 2023-08-10T06:07:08.834Z | Downstairs last five completed: 1001 1000 |
774 | 2023-08-10T06:07:08.837Z | Upstairs last five completed: 2 1 |
775 | 2023-08-10T06:07:08.840Z | CLIENT: Up:0 ds:0 act:3 |
776 | 2023-08-10T06:07:08.843Z | CLIENT: All crucible jobs finished, exiting program |
777 | 2023-08-10T06:07:08.846Z | Repair part 2 passed |
778 | 2023-08-10T06:07:08.849Z | /var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 |
779 | 2023-08-10T06:07:08.853Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:08.969861028Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1203} |
780 | 2023-08-10T06:07:08.856Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:08.970227069Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1203} |
781 | 2023-08-10T06:07:08.858Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:08.970254501Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1203} |
782 | 2023-08-10T06:07:08.862Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:08.974406945Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1203} |
783 | 2023-08-10T06:07:08.865Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:08.974458608Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1203} |
784 | 2023-08-10T06:07:08.868Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:08.974501969Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1203} |
785 | 2023-08-10T06:07:08.871Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:08.976324458Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1203} |
786 | 2023-08-10T06:07:08.873Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:08.976374583Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1203} |
787 | 2023-08-10T06:07:08.876Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:08.976417385Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1203} |
788 | 2023-08-10T06:07:08.879Z | EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF |
789 | 2023-08-10T06:07:08.882Z | 0 000-009 26 26 26 2017 2017 2017 F F F |
790 | 2023-08-10T06:07:08.884Z | 1 010-019 27 27 27 2018 2018 2018 F F F |
791 | 2023-08-10T06:07:08.887Z | 2 020-029 27 27 27 2018 2018 2018 F F F |
792 | 2023-08-10T06:07:08.890Z | 3 030-039 27 27 27 2018 2018 2018 F F F |
793 | 2023-08-10T06:07:08.893Z | 4 040-049 27 27 27 2018 2018 2018 F F F |
794 | 2023-08-10T06:07:08.896Z | Max gen: 27, Max flush: 2018 |
795 | 2023-08-10T06:07:08.899Z | dump test passed |
796 | 2023-08-10T06:07:08.902Z | /var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -e 1 |
797 | 2023-08-10T06:07:08.905Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.000379047Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
798 | 2023-08-10T06:07:08.908Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.000674855Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
799 | 2023-08-10T06:07:08.910Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.00072579Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
800 | 2023-08-10T06:07:08.914Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.004120976Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
801 | 2023-08-10T06:07:08.917Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.004184449Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
802 | 2023-08-10T06:07:08.920Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.004229234Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
803 | 2023-08-10T06:07:08.923Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.005770703Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
804 | 2023-08-10T06:07:08.926Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.005825959Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
805 | 2023-08-10T06:07:08.929Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.005869014Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
806 | 2023-08-10T06:07:08.932Z | Extent 1 |
807 | 2023-08-10T06:07:08.934Z | GEN 27 27 27 |
808 | 2023-08-10T06:07:08.937Z | FLUSH_ID 2018 2018 2018 |
809 | 2023-08-10T06:07:08.940Z | DIRTY |
810 | 2023-08-10T06:07:08.943Z | |
811 | 2023-08-10T06:07:08.946Z | BLOCK D0 D1 D2 C0 C1 C2 DIFF |
812 | 2023-08-10T06:07:08.949Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.007539201Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
813 | 2023-08-10T06:07:08.951Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.007590011Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
814 | 2023-08-10T06:07:08.955Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.007650388Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
815 | 2023-08-10T06:07:08.958Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.009422762Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
816 | 2023-08-10T06:07:08.961Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.009476278Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
817 | 2023-08-10T06:07:08.964Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.009519313Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
818 | 2023-08-10T06:07:08.967Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.011073463Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
819 | 2023-08-10T06:07:08.969Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.011144722Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
820 | 2023-08-10T06:07:08.972Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.011197054Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
821 | 2023-08-10T06:07:08.975Z | 010 A A A A A A |
822 | 2023-08-10T06:07:08.978Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.013310143Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
823 | 2023-08-10T06:07:08.981Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.013362268Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
824 | 2023-08-10T06:07:08.984Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.013405241Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
825 | 2023-08-10T06:07:08.987Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.015005893Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
826 | 2023-08-10T06:07:08.990Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.015058886Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
827 | 2023-08-10T06:07:08.993Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.015101646Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
828 | 2023-08-10T06:07:08.996Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.016635188Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
829 | 2023-08-10T06:07:08.999Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.016689922Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
830 | 2023-08-10T06:07:09.002Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.016733008Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
831 | 2023-08-10T06:07:09.005Z | 011 A A A A A A |
832 | 2023-08-10T06:07:09.008Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.018290063Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
833 | 2023-08-10T06:07:09.011Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.018341758Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
834 | 2023-08-10T06:07:09.014Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.018363929Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
835 | 2023-08-10T06:07:09.017Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.019908648Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
836 | 2023-08-10T06:07:09.020Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.019960724Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
837 | 2023-08-10T06:07:09.023Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.020003326Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
838 | 2023-08-10T06:07:09.025Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.021515678Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
839 | 2023-08-10T06:07:09.028Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.021566885Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
840 | 2023-08-10T06:07:09.031Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.02160992Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
841 | 2023-08-10T06:07:09.034Z | 012 A A A A A A |
842 | 2023-08-10T06:07:09.036Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.023183209Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
843 | 2023-08-10T06:07:09.039Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.023238431Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
844 | 2023-08-10T06:07:09.042Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.023281468Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
845 | 2023-08-10T06:07:09.045Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.024790552Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
846 | 2023-08-10T06:07:09.048Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.024841505Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
847 | 2023-08-10T06:07:09.050Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.024884076Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
848 | 2023-08-10T06:07:09.053Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.026397123Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
849 | 2023-08-10T06:07:09.056Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.026450244Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
850 | 2023-08-10T06:07:09.059Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.026493275Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
851 | 2023-08-10T06:07:09.062Z | 013 A A A A A A |
852 | 2023-08-10T06:07:09.065Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.028103482Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
853 | 2023-08-10T06:07:09.067Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.028175726Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
854 | 2023-08-10T06:07:09.070Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.028223046Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
855 | 2023-08-10T06:07:09.073Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.029698838Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
856 | 2023-08-10T06:07:09.076Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.029751197Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
857 | 2023-08-10T06:07:09.079Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.02979371Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
858 | 2023-08-10T06:07:09.082Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.031307156Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
859 | 2023-08-10T06:07:09.085Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.031359611Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
860 | 2023-08-10T06:07:09.088Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.031402444Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
861 | 2023-08-10T06:07:09.091Z | 014 A A A A A A |
862 | 2023-08-10T06:07:09.094Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.032959234Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
863 | 2023-08-10T06:07:09.096Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.033011409Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
864 | 2023-08-10T06:07:09.100Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.033054042Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
865 | 2023-08-10T06:07:09.102Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.034598627Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
866 | 2023-08-10T06:07:09.105Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.034650373Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
867 | 2023-08-10T06:07:09.108Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.034672231Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
868 | 2023-08-10T06:07:09.111Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.03621075Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
869 | 2023-08-10T06:07:09.114Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.036241067Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
870 | 2023-08-10T06:07:09.117Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.036259858Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
871 | 2023-08-10T06:07:09.119Z | 015 A A A A A A |
872 | 2023-08-10T06:07:09.122Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.03784644Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
873 | 2023-08-10T06:07:09.125Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.03787447Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
874 | 2023-08-10T06:07:09.128Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.037893408Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
875 | 2023-08-10T06:07:09.131Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.039480373Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
876 | 2023-08-10T06:07:09.134Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.03950722Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
877 | 2023-08-10T06:07:09.136Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.039526382Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
878 | 2023-08-10T06:07:09.139Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.041095368Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
879 | 2023-08-10T06:07:09.142Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.041126183Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
880 | 2023-08-10T06:07:09.145Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.041158411Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
881 | 2023-08-10T06:07:09.148Z | 016 A A A A A A |
882 | 2023-08-10T06:07:09.151Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.042783451Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
883 | 2023-08-10T06:07:09.154Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.042810538Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
884 | 2023-08-10T06:07:09.157Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.042828986Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
885 | 2023-08-10T06:07:09.160Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.044397892Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
886 | 2023-08-10T06:07:09.162Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.044426676Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
887 | 2023-08-10T06:07:09.165Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.044444897Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
888 | 2023-08-10T06:07:09.168Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.046002866Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
889 | 2023-08-10T06:07:09.171Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.046029564Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
890 | 2023-08-10T06:07:09.173Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.046048641Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
891 | 2023-08-10T06:07:09.176Z | 017 A A A A A A |
892 | 2023-08-10T06:07:09.181Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.04764313Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
893 | 2023-08-10T06:07:09.185Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.047673963Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
894 | 2023-08-10T06:07:09.188Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.047693305Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
895 | 2023-08-10T06:07:09.190Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.049254867Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
896 | 2023-08-10T06:07:09.193Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.049283369Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
897 | 2023-08-10T06:07:09.196Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.049302542Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
898 | 2023-08-10T06:07:09.199Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.050856167Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
899 | 2023-08-10T06:07:09.202Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.050884076Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
900 | 2023-08-10T06:07:09.204Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.050902906Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
901 | 2023-08-10T06:07:09.207Z | 018 A A A A A A |
902 | 2023-08-10T06:07:09.210Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.052485866Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
903 | 2023-08-10T06:07:09.213Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.052523684Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
904 | 2023-08-10T06:07:09.216Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.052544591Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
905 | 2023-08-10T06:07:09.219Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.054095663Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
906 | 2023-08-10T06:07:09.221Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.054122479Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
907 | 2023-08-10T06:07:09.224Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.054152595Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
908 | 2023-08-10T06:07:09.227Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.0557529Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
909 | 2023-08-10T06:07:09.230Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.0557826Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
910 | 2023-08-10T06:07:09.233Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.055801619Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1204} |
911 | 2023-08-10T06:07:09.236Z | 019 A A A A A A |
912 | 2023-08-10T06:07:09.239Z | dump extent test passed |
913 | 2023-08-10T06:07:09.241Z | /var/tmp/bins/crucible-downstairs dump --no-color -d /var/tmp/test_up/8810 -d /var/tmp/test_up/8820 -d /var/tmp/test_up/8830 -b 20 |
914 | 2023-08-10T06:07:09.244Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.08151056Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
915 | 2023-08-10T06:07:09.247Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.081862563Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
916 | 2023-08-10T06:07:09.249Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.081887724Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
917 | 2023-08-10T06:07:09.252Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.085407806Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
918 | 2023-08-10T06:07:09.254Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.085439357Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
919 | 2023-08-10T06:07:09.257Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.085457975Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
920 | 2023-08-10T06:07:09.260Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.087122383Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
921 | 2023-08-10T06:07:09.263Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.087209431Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
922 | 2023-08-10T06:07:09.265Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.087231091Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
923 | 2023-08-10T06:07:09.268Z | Extent 2 Block in extent 0 Actual block 20 |
924 | 2023-08-10T06:07:09.271Z | |
925 | 2023-08-10T06:07:09.273Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.088735335Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
926 | 2023-08-10T06:07:09.446Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.088762948Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
927 | 2023-08-10T06:07:09.449Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.088781842Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
928 | 2023-08-10T06:07:09.452Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.090640783Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
929 | 2023-08-10T06:07:09.455Z | |
930 | 2023-08-10T06:07:09.458Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.090671305Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
931 | 2023-08-10T06:07:09.461Z | real 5:43.055955898 |
932 | 2023-08-10T06:07:09.464Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.090689846Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
933 | 2023-08-10T06:07:09.467Z | user 4:36.373513163 |
934 | 2023-08-10T06:07:09.469Z | {"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.092279756Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
935 | 2023-08-10T06:07:09.472Z | sys 23.444112608 |
936 | 2023-08-10T06:07:09.475Z | {"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.092306403Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
937 | 2023-08-10T06:07:09.477Z | trap 0.330674163 |
938 | 2023-08-10T06:07:09.480Z | {"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-08-10T06:07:09.092325003Z","hostname":"ip-10-150-1-162.us-west-2.compute.internal","pid":1205} |
939 | 2023-08-10T06:07:09.483Z | tflt 0.148079149 |
940 | 2023-08-10T06:07:09.486Z | DATA SHA256 VER |
941 | 2023-08-10T06:07:09.488Z | dflt 0.047232643 |
942 | 2023-08-10T06:07:09.491Z | ------ ---------------------------------------------------------------- --- |
943 | 2023-08-10T06:07:09.494Z | kflt 0.001212589 |
944 | 2023-08-10T06:07:09.497Z | 0 b4f5ee008c8fe786954d733e17bd9197c20450ac551bda64d79e60d9979d2a8b A |
945 | 2023-08-10T06:07:09.499Z | lock 53:06.385727177 |
946 | 2023-08-10T06:07:09.502Z | 1 b4f5ee008c8fe786954d733e17bd9197c20450ac551bda64d79e60d9979d2a8b A |
947 | 2023-08-10T06:07:09.506Z | slp 10:46.700548274 |
948 | 2023-08-10T06:07:09.509Z | 2 b4f5ee008c8fe786954d733e17bd9197c20450ac551bda64d79e60d9979d2a8b A |
949 | 2023-08-10T06:07:09.511Z | lat 19.043094756 |
950 | 2023-08-10T06:07:09.514Z | |
951 | 2023-08-10T06:07:09.517Z | stop 0.030080998 |
952 | 2023-08-10T06:07:09.519Z | NONCES 0 1 2 DIFF |
953 | 2023-08-10T06:07:09.522Z | ------ ------------------------ ------------------------ ------------------------ ----- |
954 | 2023-08-10T06:07:09.525Z | 0 b0415b57bd56819fbefad691 b0415b57bd56819fbefad691 b0415b57bd56819fbefad691 |
955 | 2023-08-10T06:07:09.528Z | |
956 | 2023-08-10T06:07:09.530Z | TAGS 0 1 2 DIFF |
957 | 2023-08-10T06:07:09.533Z | ------ -------------------------------- -------------------------------- -------------------------------- ----- |
958 | 2023-08-10T06:07:09.536Z | 0 561a93fe052ba11f0d335bf7e9cdaee7 561a93fe052ba11f0d335bf7e9cdaee7 561a93fe052ba11f0d335bf7e9cdaee7 |
959 | 2023-08-10T06:07:09.538Z | |
960 | 2023-08-10T06:07:09.541Z | HASHES 0 1 2 DIFF |
961 | 2023-08-10T06:07:09.544Z | ------ ---------------- ---------------- ---------------- ----- |
962 | 2023-08-10T06:07:09.547Z | 0 ab94edde9aa0240e ab94edde9aa0240e ab94edde9aa0240e |
963 | 2023-08-10T06:07:09.549Z | |
964 | 2023-08-10T06:07:09.552Z | dump block test passed |
965 | 2023-08-10T06:07:09.555Z | Upstairs tests have completed, stopping all downstairs |
966 | 2023-08-10T06:07:09.557Z | |
967 | 2023-08-10T06:07:09.560Z | All Tests have passed |
968 | 2023-08-10T06:07:09.563Z | 5:44 Test duration |
969 | 2023-08-10T06:07:09.566Z | process exited: duration 355172 ms, exit code 0 |
|
970 | 2023-08-10T06:07:09.572Z | found 2 output files |
971 | 2023-08-10T06:07:09.574Z | uploading: /tmp/test_up/dsc-out.txt (4399 bytes) |
972 | 2023-08-10T06:07:09.584Z | uploaded: /tmp/test_up/dsc-out.txt |
973 | 2023-08-10T06:07:09.594Z | uploading: /tmp/test_up/test_up_out.txt (3412466 bytes) |
974 | 2023-08-10T06:07:10.515Z | uploaded: /tmp/test_up/test_up_out.txt |