01HAZHJ5489ZT7Y1V4XWMESYM5: test-crudd-benchmark

BasicConfig {
    output_rules: [
        "/tmp/crudd-speed-battery-results.json",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HAZHJES9FCBT7FNET62XHDVG

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12023-09-22T23:30:32.452Zjob dependencies complete; ready to run (waiting for 36 m 56 s)
22023-09-22T23:31:24.500Zjob assigned to worker 01HAZKP7RPZADMZF5T89WK01XX (queued for 52 s)
32023-09-22T23:31:25.156Zdownloading input: /input/rbuild/out/crucible-nightly.sha256.txt
42023-09-22T23:31:25.191Zdownloaded input: /input/rbuild/out/crucible-nightly.sha256.txt
52023-09-22T23:31:25.197Zdownloading input: /input/rbuild/out/crucible-nightly.tar.gz
62023-09-22T23:31:36.798Zdownloaded input: /input/rbuild/out/crucible-nightly.tar.gz
72023-09-22T23:31:36.802Zdownloading input: /input/rbuild/out/crucible-pantry.sha256.txt
82023-09-22T23:31:36.835Zdownloaded input: /input/rbuild/out/crucible-pantry.sha256.txt
92023-09-22T23:31:36.839Zdownloading input: /input/rbuild/out/crucible-pantry.tar.gz
102023-09-22T23:31:41.825Zdownloaded input: /input/rbuild/out/crucible-pantry.tar.gz
112023-09-22T23:31:41.837Zdownloading input: /input/rbuild/out/crucible.sha256.txt
122023-09-22T23:31:41.847Zdownloaded input: /input/rbuild/out/crucible.sha256.txt
132023-09-22T23:31:41.852Zdownloading input: /input/rbuild/out/crucible.tar.gz
142023-09-22T23:31:46.900Zdownloaded input: /input/rbuild/out/crucible.tar.gz
152023-09-22T23:31:46.905Zdownloading input: /input/rbuild/work/rbins/crucible-downstairs.gz
162023-09-22T23:31:49.645Zdownloaded input: /input/rbuild/work/rbins/crucible-downstairs.gz
172023-09-22T23:31:49.651Zdownloading input: /input/rbuild/work/rbins/crucible-hammer.gz
182023-09-22T23:31:52.067Zdownloaded input: /input/rbuild/work/rbins/crucible-hammer.gz
192023-09-22T23:31:52.070Zdownloading input: /input/rbuild/work/rbins/crudd.gz
202023-09-22T23:31:56.491Zdownloaded input: /input/rbuild/work/rbins/crudd.gz
212023-09-22T23:31:56.494Zdownloading input: /input/rbuild/work/rbins/crutest.gz
222023-09-22T23:32:00.695Zdownloaded input: /input/rbuild/work/rbins/crutest.gz
232023-09-22T23:32:00.699Zdownloading input: /input/rbuild/work/rbins/dsc.gz
242023-09-22T23:32:04.280Zdownloaded input: /input/rbuild/work/rbins/dsc.gz
252023-09-22T23:32:04.283Zdownloading input: /input/rbuild/work/scripts/crudd-speed-battery.sh
262023-09-22T23:32:04.299Zdownloaded input: /input/rbuild/work/scripts/crudd-speed-battery.sh
272023-09-22T23:32:04.303Zdownloading input: /input/rbuild/work/scripts/perf-downstairs-tick.d
282023-09-22T23:32:04.316Zdownloaded input: /input/rbuild/work/scripts/perf-downstairs-tick.d
292023-09-22T23:32:04.318Zdownloading input: /input/rbuild/work/scripts/test_mem.sh
302023-09-22T23:32:04.331Zdownloaded input: /input/rbuild/work/scripts/test_mem.sh
312023-09-22T23:32:04.334Zdownloading input: /input/rbuild/work/scripts/test_perf.sh
322023-09-22T23:32:04.347Zdownloaded input: /input/rbuild/work/scripts/test_perf.sh
332023-09-22T23:32:04.351Zdownloading input: /input/rbuild/tmp/cargo-test-out.log
342023-09-22T23:32:04.706Zdownloaded input: /input/rbuild/tmp/cargo-test-out.log
 
352023-09-22T23:32:04.709Zstarting task 0: "setup"
362023-09-22T23:32:04.773Z++ uname -s
372023-09-22T23:32:04.779Z+ kern=SunOS
382023-09-22T23:32:04.785Z+ case "$kern" in
392023-09-22T23:32:04.790Z+ groupadd -g 12345 build
402023-09-22T23:32:04.798Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
412023-09-22T23:32:06.784Z+ zfs create -o mountpoint=/work rpool/work
422023-09-22T23:32:06.883Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
432023-09-22T23:32:06.893Z+ home_fs=zfs
442023-09-22T23:32:06.900Z+ [[ zfs == autofs ]]
452023-09-22T23:32:06.910Z+ mkdir -p /home/build
462023-09-22T23:32:06.916Z+ chown build:build /home/build /work
472023-09-22T23:32:08.887Z+ chmod 0700 /home/build /work
482023-09-22T23:32:08.891Zprocess exited: duration 4177 ms, exit code 0
 
492023-09-22T23:32:08.897Zstarting task 1: "authentication"
502023-09-22T23:32:08.925Zprocess exited: duration 24 ms, exit code 0
 
512023-09-22T23:32:08.931Zstarting task 2: "build"
522023-09-22T23:32:08.943Z+ banner cores
532023-09-22T23:32:08.961Z
542023-09-22T23:32:08.964Z #### #### ##### ###### ####
552023-09-22T23:32:08.968Z # # # # # # # #
562023-09-22T23:32:08.972Z # # # # # ##### ####
572023-09-22T23:32:08.975Z # # # ##### # #
582023-09-22T23:32:08.977Z # # # # # # # # #
592023-09-22T23:32:08.980Z #### #### # # ###### ####
602023-09-22T23:32:08.983Z
612023-09-22T23:32:08.986Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
622023-09-22T23:32:08.989Z+ echo 'input rbins dir contains:'
632023-09-22T23:32:08.992Zinput rbins dir contains:
642023-09-22T23:32:08.995Z+ ls -ltr /input/rbuild/work/rbins
652023-09-22T23:32:08.998Ztotal 181304
662023-09-22T23:32:09.001Z-rw-r--r-- 1 root root 23383252 Sep 22 23:31 crucible-downstairs.gz
672023-09-22T23:32:09.004Z-rw-r--r-- 1 root root 18774941 Sep 22 23:31 crucible-hammer.gz
682023-09-22T23:32:09.007Z-rw-r--r-- 1 root root 17768782 Sep 22 23:31 crudd.gz
692023-09-22T23:32:09.009Z-rw-r--r-- 1 root root 20586550 Sep 22 23:31 crutest.gz
702023-09-22T23:32:09.013Z-rw-r--r-- 1 root root 12242158 Sep 22 23:32 dsc.gz
712023-09-22T23:32:09.016Z+ echo 'input scripts dir contains:'
722023-09-22T23:32:09.019Zinput scripts dir contains:
732023-09-22T23:32:09.022Z+ ls -ltr /input/rbuild/work/scripts
742023-09-22T23:32:09.025Ztotal 25
752023-09-22T23:32:09.027Z-rw-r--r-- 1 root root 7480 Sep 22 23:32 crudd-speed-battery.sh
762023-09-22T23:32:09.031Z-rw-r--r-- 1 root root 3056 Sep 22 23:32 perf-downstairs-tick.d
772023-09-22T23:32:09.055Z-rw-r--r-- 1 root root 5801 Sep 22 23:32 test_mem.sh
782023-09-22T23:32:09.058Z-rw-r--r-- 1 root root 5171 Sep 22 23:32 test_perf.sh
792023-09-22T23:32:09.061Z+ banner unpack
802023-09-22T23:32:09.064Z
812023-09-22T23:32:09.067Z # # # # ##### ## #### # #
822023-09-22T23:32:09.070Z # # ## # # # # # # # # #
832023-09-22T23:32:09.073Z # # # # # # # # # # ####
842023-09-22T23:32:09.075Z # # # # # ##### ###### # # #
852023-09-22T23:32:09.078Z # # # ## # # # # # # #
862023-09-22T23:32:09.081Z #### # # # # # #### # #
872023-09-22T23:32:09.084Z
882023-09-22T23:32:09.089Z+ mkdir -p /var/tmp/bins
892023-09-22T23:32:09.091Z+ for t in "$input/rbins/"*.gz
902023-09-22T23:32:09.094Z++ basename /input/rbuild/work/rbins/crucible-downstairs.gz
912023-09-22T23:32:09.097Z+ b=crucible-downstairs.gz
922023-09-22T23:32:09.100Z+ b=crucible-downstairs
932023-09-22T23:32:09.103Z+ gunzip
942023-09-22T23:32:09.684Z+ chmod +x /var/tmp/bins/crucible-downstairs
952023-09-22T23:32:09.687Z+ for t in "$input/rbins/"*.gz
962023-09-22T23:32:09.692Z++ basename /input/rbuild/work/rbins/crucible-hammer.gz
972023-09-22T23:32:09.695Z+ b=crucible-hammer.gz
982023-09-22T23:32:09.698Z+ b=crucible-hammer
992023-09-22T23:32:09.702Z+ gunzip
1002023-09-22T23:32:10.236Z+ chmod +x /var/tmp/bins/crucible-hammer
1012023-09-22T23:32:10.241Z+ for t in "$input/rbins/"*.gz
1022023-09-22T23:32:10.245Z++ basename /input/rbuild/work/rbins/crudd.gz
1032023-09-22T23:32:10.248Z+ b=crudd.gz
1042023-09-22T23:32:10.251Z+ b=crudd
1052023-09-22T23:32:10.254Z+ gunzip
1062023-09-22T23:32:10.757Z+ chmod +x /var/tmp/bins/crudd
1072023-09-22T23:32:10.761Z+ for t in "$input/rbins/"*.gz
1082023-09-22T23:32:10.764Z++ basename /input/rbuild/work/rbins/crutest.gz
1092023-09-22T23:32:10.767Z+ b=crutest.gz
1102023-09-22T23:32:10.770Z+ b=crutest
1112023-09-22T23:32:10.773Z+ gunzip
1122023-09-22T23:32:11.360Z+ chmod +x /var/tmp/bins/crutest
1132023-09-22T23:32:11.363Z+ for t in "$input/rbins/"*.gz
1142023-09-22T23:32:11.366Z++ basename /input/rbuild/work/rbins/dsc.gz
1152023-09-22T23:32:11.369Z+ b=dsc.gz
1162023-09-22T23:32:11.373Z+ b=dsc
1172023-09-22T23:32:11.376Z+ gunzip
1182023-09-22T23:32:11.723Z+ chmod +x /var/tmp/bins/dsc
1192023-09-22T23:32:11.727Z+ export BINDIR=/var/tmp/bins
1202023-09-22T23:32:11.730Z+ BINDIR=/var/tmp/bins
1212023-09-22T23:32:11.733Z+ banner 'crudd bench'
1222023-09-22T23:32:11.736Z
1232023-09-22T23:32:11.739Z #### ##### # # ##### ##### ##### ###### # # ####
1242023-09-22T23:32:11.742Z # # # # # # # # # # # # # ## # # #
1252023-09-22T23:32:11.744Z # # # # # # # # # ##### ##### # # # #
1262023-09-22T23:32:11.747Z # ##### # # # # # # # # # # # # #
1272023-09-22T23:32:11.751Z # # # # # # # # # # # # # # ## # #
1282023-09-22T23:32:11.754Z #### # # #### ##### ##### ##### ###### # # ####
1292023-09-22T23:32:11.757Z
1302023-09-22T23:32:11.760Z+ pfexec plimit -n 9123456 1125
1312023-09-22T23:32:11.763Z+ ptime -m bash /input/rbuild/work/scripts/crudd-speed-battery.sh
1322023-09-22T23:32:11.844Zrun single test
1332023-09-22T23:32:11.865ZCreating region directory at: /var/tmp/dsc/region
1342023-09-22T23:32:11.905ZCreating dsc directory at: /tmp/dsc
1352023-09-22T23:32:12.944ZDownstairs region 0 created at /var/tmp/dsc/region/8810 in 1.0681472
1362023-09-22T23:32:12.990ZDownstairs region 1 created at /var/tmp/dsc/region/8820 in 0.054720864
1372023-09-22T23:32:13.043ZDownstairs region 2 created at /var/tmp/dsc/region/8830 in 0.053836368
1382023-09-22T23:32:13.046ZRegion set with 3 regions was created
1392023-09-22T23:32:13.056ZUsing existing output directory "/tmp/dsc"
1402023-09-22T23:32:13.059Zstart ds: 8810
1412023-09-22T23:32:13.062Zstart ds: 8820
1422023-09-22T23:32:13.065Zstart ds: 8830
1432023-09-22T23:32:13.068Zstart access at:127.0.0.1:9998
1442023-09-22T23:32:13.071ZStarting downstairs at port 8810
1452023-09-22T23:32:13.074ZMake output file at "/tmp/dsc/downstairs-8810.txt"
1462023-09-22T23:32:13.077ZStarting downstairs at port 8830
1472023-09-22T23:32:13.080ZMake output file at "/tmp/dsc/downstairs-8830.txt"
1482023-09-22T23:32:13.084Z[8810][0] reports Starting
1492023-09-22T23:32:13.087Z[8830][2] reports Starting
1502023-09-22T23:32:13.090ZStarting downstairs at port 8820
1512023-09-22T23:32:13.093ZMake output file at "/tmp/dsc/downstairs-8820.txt"
1522023-09-22T23:32:13.096Z[8820][1] reports Starting
1532023-09-22T23:32:13.099ZDownstairs /var/tmp/dsc/region/8830 port 8830 PID:Some(1164)
1542023-09-22T23:32:13.102Z[8830][2] reports Running
1552023-09-22T23:32:13.105ZDownstairs /var/tmp/dsc/region/8820 port 8820 PID:Some(1165)
1562023-09-22T23:32:13.107Z[8820][1] reports Running
1572023-09-22T23:32:13.111ZDownstairs /var/tmp/dsc/region/8810 port 8810 PID:Some(1166)
1582023-09-22T23:32:13.113Z[8810][0] reports Running
1592023-09-22T23:32:13.116ZSep 22 23:32:11.874 INFO listening, local_addr: 127.0.0.1:9998
1602023-09-22T23:32:13.140ZControl access at:127.0.0.1:9998
1612023-09-22T23:32:16.056ZStarted DSC 1162
1622023-09-22T23:32:16.210Zraw options: Opt { target: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], key: None, gen: 1, cert_pem: None, key_pem: None, root_cert_pem: None, control: None, num_bytes: Some(4294967296), byte_offset: 0, iocmd_block_count: 4096, pipeline_length: 8, benchmarking_mode: Some("/tmp/crudd-benchmark-num-bytes-processed.txt"), subcommand: Read }
1632023-09-22T23:32:16.214Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.017995897Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1642023-09-22T23:32:16.217Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: false,\n opt_level: 3,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.018305295Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1652023-09-22T23:32:16.220Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.018332376Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1662023-09-22T23:32:16.223Z{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.018340679Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1672023-09-22T23:32:16.226Z{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: d59e951a-abd0-4242-81a9-ca75d96f77ae","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.018347527Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1682023-09-22T23:32:16.229ZCrucible runtime is spawned
1692023-09-22T23:32:16.232ZThe guest has requested activation
1702023-09-22T23:32:16.234Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.018960209Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"task":"up_listen"}
1712023-09-22T23:32:16.237Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019044078Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"looper":"1"}
1722023-09-22T23:32:16.239Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019057998Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"looper":"2"}
1732023-09-22T23:32:16.242Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019087951Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"looper":"0"}
1742023-09-22T23:32:16.245Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019095234Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1752023-09-22T23:32:16.248Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019101975Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1762023-09-22T23:32:16.251Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019315154Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"looper":"0"}
1772023-09-22T23:32:16.254Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.0193766Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"looper":"1"}
1782023-09-22T23:32:16.256Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019389555Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"looper":"2"}
1792023-09-22T23:32:16.259Z{"msg":"00000000-0000-0000-0000-000000000000 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019400717Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1802023-09-22T23:32:16.262Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019581162Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1812023-09-22T23:32:16.265Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019621966Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1822023-09-22T23:32:16.268Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.019913359Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1832023-09-22T23:32:16.271Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (0f45cb77-94d1-451a-b309-ed315932e6f9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.020755952Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1842023-09-22T23:32:16.274Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.020799551Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1852023-09-22T23:32:16.277Z{"msg":"[2] client is_active_req TRUE, promote! session 0f45cb77-94d1-451a-b309-ed315932e6f9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02088487Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1862023-09-22T23:32:16.280Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (0f45cb77-94d1-451a-b309-ed315932e6f9) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.020929506Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1872023-09-22T23:32:16.282Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.020941191Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1882023-09-22T23:32:16.285Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (0f45cb77-94d1-451a-b309-ed315932e6f9) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.020951567Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1892023-09-22T23:32:16.288Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.020961997Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1902023-09-22T23:32:16.291Z{"msg":"[0] client is_active_req TRUE, promote! session 0f45cb77-94d1-451a-b309-ed315932e6f9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.020972587Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1912023-09-22T23:32:16.293Z{"msg":"[1] client is_active_req TRUE, promote! session 0f45cb77-94d1-451a-b309-ed315932e6f9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.020984049Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1922023-09-22T23:32:16.296Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.021227787Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1932023-09-22T23:32:16.299Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.021327242Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1942023-09-22T23:32:16.302Z{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02134219Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1952023-09-22T23:32:16.305Z{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.021376955Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1962023-09-22T23:32:16.309Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.021387398Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1972023-09-22T23:32:16.311Z{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.021396428Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1982023-09-22T23:32:16.314Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.021405429Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
1992023-09-22T23:32:16.317Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.021438539Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2002023-09-22T23:32:16.320Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02144897Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2012023-09-22T23:32:16.323Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (0f45cb77-94d1-451a-b309-ed315932e6f9) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02309863Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2022023-09-22T23:32:16.326Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023118049Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2032023-09-22T23:32:16.329Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:15.023129228Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2042023-09-22T23:32:16.332Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023210238Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2052023-09-22T23:32:16.335Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (0f45cb77-94d1-451a-b309-ed315932e6f9) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023247796Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2062023-09-22T23:32:16.338Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023266031Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2072023-09-22T23:32:16.341Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:15.02330231Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2082023-09-22T23:32:16.344Z{"msg":"00000000-0000-0000-0000-000000000000 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023313112Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2092023-09-22T23:32:16.346Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (0f45cb77-94d1-451a-b309-ed315932e6f9) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023346131Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2102023-09-22T23:32:16.349Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023356718Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2112023-09-22T23:32:16.352Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:15.02339886Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2122023-09-22T23:32:16.356Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02341027Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2132023-09-22T23:32:16.360Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023420434Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2142023-09-22T23:32:16.363Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023431722Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2152023-09-22T23:32:16.366Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023467914Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2162023-09-22T23:32:16.369Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023482097Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2172023-09-22T23:32:16.373Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02351789Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2182023-09-22T23:32:16.375Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023530169Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2192023-09-22T23:32:16.378Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023540887Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2202023-09-22T23:32:16.381Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023573715Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2212023-09-22T23:32:16.384Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023584248Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2222023-09-22T23:32:16.386Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023621744Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2232023-09-22T23:32:16.389Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023632401Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2242023-09-22T23:32:16.392Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":The guest has finished waiting for activation
2252023-09-22T23:32:16.395Z30,"time":"2023-09-22T23:32:15.02366953Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2262023-09-22T23:32:16.398Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023681802Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2272023-09-22T23:32:16.401Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023716267Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2282023-09-22T23:32:16.404Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023727722Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2292023-09-22T23:32:16.407Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023768528Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2302023-09-22T23:32:16.410Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023779738Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2312023-09-22T23:32:16.413Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023791363Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2322023-09-22T23:32:16.416Z{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 0f45cb77-94d1-451a-b309-ed315932e6f9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023801417Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2332023-09-22T23:32:16.419Z{"msg":"00000000-0000-0000-0000-000000000000 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023810987Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2342023-09-22T23:32:16.422Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023820618Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2352023-09-22T23:32:16.425Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02383005Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2362023-09-22T23:32:16.431Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023840448Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2372023-09-22T23:32:16.439Z{"msg":"00000000-0000-0000-0000-000000000000 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023850858Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2382023-09-22T23:32:16.444Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02386093Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2392023-09-22T23:32:16.448Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023870109Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2402023-09-22T23:32:16.451Z{"msg":"00000000-0000-0000-0000-000000000000 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023879534Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2412023-09-22T23:32:16.454Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023889894Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2422023-09-22T23:32:16.458Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02390004Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2432023-09-22T23:32:16.464Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023909654Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2442023-09-22T23:32:16.470Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023918302Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2452023-09-22T23:32:16.482Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023924282Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2462023-09-22T23:32:16.487Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02393002Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2472023-09-22T23:32:16.490Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023935727Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2482023-09-22T23:32:16.493Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023941503Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2492023-09-22T23:32:16.496Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.02394703Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2502023-09-22T23:32:16.499Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:15.023952694Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2512023-09-22T23:32:17.233Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.024879103Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2522023-09-22T23:32:17.236Z{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.024910223Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2532023-09-22T23:33:00.175Z{"msg":"Request to deactivate this guest","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.982095105Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2542023-09-22T23:33:00.179Z{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.983956841Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2552023-09-22T23:33:00.182Z{"msg":"[1] deactivate job 3134 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.984120241Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2562023-09-22T23:33:00.185Z{"msg":"[2] deactivate job 3135 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985146691Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2572023-09-22T23:33:00.188Z{"msg":"[0] deactivate job 3135 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.98517504Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2582023-09-22T23:33:00.191Z{"msg":"[1] deactivate job 3135 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985181598Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2592023-09-22T23:33:00.195Z{"msg":"[2] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.98518798Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2602023-09-22T23:33:00.198Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (0f45cb77-94d1-451a-b309-ed315932e6f9) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985194736Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2612023-09-22T23:33:00.200Z{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.98520153Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2622023-09-22T23:33:00.203Z{"msg":"[0] deactivate, no work so YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985354418Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2632023-09-22T23:33:00.206Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (0f45cb77-94d1-451a-b309-ed315932e6f9) Active Active Deactivated ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985368365Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2642023-09-22T23:33:00.209Z{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985377783Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2652023-09-22T23:33:00.212Z{"msg":"127.0.0.1:8810: proc: [0] client work task ended, Ok(Err([0] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:32:58.985627376Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"looper":"0"}
2662023-09-22T23:33:00.215Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985672458Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2672023-09-22T23:33:00.217Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985685273Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2682023-09-22T23:33:00.220Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985695033Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2692023-09-22T23:33:00.223Z{"msg":"127.0.0.1:8830: proc: [2] client work task ended, Ok(Err([2] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:32:58.985705278Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"looper":"2"}
2702023-09-22T23:33:00.226Z{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985717877Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2712023-09-22T23:33:00.229Z{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985728156Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171}
2722023-09-22T23:33:00.232Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8810 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:58.985738385Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"looper":"0"}
2732023-09-22T23:33:00.235Z{"msg":"127.0.0.1:8810 Message to ds_status_tx failed: SendError { .. }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:32:58.985749438Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1171,"looper":"0"}
2742023-09-22T23:33:00.237Zraw options: Opt { target: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], key: None, gen: 2, cert_pem: None, key_pem: None, root_cert_pem: None, control: None, num_bytes: Some(4294967296), byte_offset: 0, iocmd_block_count: 4096, pipeline_length: 8, benchmarking_mode: Some("/tmp/crudd-benchmark-num-bytes-processed.txt"), subcommand: Write }
2752023-09-22T23:33:00.241Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.03698285Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2762023-09-22T23:33:00.244Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: false,\n opt_level: 3,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037334457Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2772023-09-22T23:33:00.246Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037368514Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2782023-09-22T23:33:00.249Z{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037379635Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2792023-09-22T23:33:00.253Z{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 7cc768e3-2fda-46f3-ae9a-191bdaeb777c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037390305Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2802023-09-22T23:33:00.255ZCrucible runtime is spawned
2812023-09-22T23:33:00.258ZThe guest has requested activation
2822023-09-22T23:33:00.261Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037539608Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"task":"up_listen"}
2832023-09-22T23:33:00.264Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037585246Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"0"}
2842023-09-22T23:33:00.267Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037599091Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2852023-09-22T23:33:00.270Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037635004Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2862023-09-22T23:33:00.273Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.03770875Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"1"}
2872023-09-22T23:33:00.276Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037762333Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"0"}
2882023-09-22T23:33:00.279Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037824227Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"2"}
2892023-09-22T23:33:00.282Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037877276Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"1"}
2902023-09-22T23:33:00.285Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.037977429Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"2"}
2912023-09-22T23:33:00.288Z{"msg":"00000000-0000-0000-0000-000000000000 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038083244Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2922023-09-22T23:33:00.291Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038159183Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2932023-09-22T23:33:00.294Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038271636Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2942023-09-22T23:33:00.297Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038359864Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2952023-09-22T23:33:00.300Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (ca1663e2-ea68-4cd9-8d69-29a1edc38cb6) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038472879Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2962023-09-22T23:33:00.303Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038507169Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2972023-09-22T23:33:00.308Z{"msg":"[1] client is_active_req TRUE, promote! session ca1663e2-ea68-4cd9-8d69-29a1edc38cb6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038514849Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2982023-09-22T23:33:00.311Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (ca1663e2-ea68-4cd9-8d69-29a1edc38cb6) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038521092Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
2992023-09-22T23:33:00.314Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.03852706Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3002023-09-22T23:33:00.318Z{"msg":"[0] client is_active_req TRUE, promote! session ca1663e2-ea68-4cd9-8d69-29a1edc38cb6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038533352Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3012023-09-22T23:33:00.321Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (ca1663e2-ea68-4cd9-8d69-29a1edc38cb6) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038539697Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3022023-09-22T23:33:00.324Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038545827Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3032023-09-22T23:33:00.327Z{"msg":"[2] client is_active_req TRUE, promote! session ca1663e2-ea68-4cd9-8d69-29a1edc38cb6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038551923Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3042023-09-22T23:33:00.330Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038761857Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3052023-09-22T23:33:00.333Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038774829Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3062023-09-22T23:33:00.336Z{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038781678Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3072023-09-22T23:33:00.339Z{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038788175Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3082023-09-22T23:33:00.342Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038799491Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3092023-09-22T23:33:00.345Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.03881537Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3102023-09-22T23:33:00.347Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038833016Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3112023-09-22T23:33:00.350Z{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.038862023Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3122023-09-22T23:33:00.353Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.03889064Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3132023-09-22T23:33:00.356Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (ca1663e2-ea68-4cd9-8d69-29a1edc38cb6) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.039965039Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3142023-09-22T23:33:00.358Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040003498Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3152023-09-22T23:33:00.361Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:59.040011938Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3162023-09-22T23:33:00.384Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (ca1663e2-ea68-4cd9-8d69-29a1edc38cb6) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040018085Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3172023-09-22T23:33:00.388Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040026879Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3182023-09-22T23:33:00.390Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.04003788Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3192023-09-22T23:33:00.393Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:59.040065298Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3202023-09-22T23:33:00.396Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040071306Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3212023-09-22T23:33:00.399Z{"msg":"00000000-0000-0000-0000-000000000000 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040077569Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3222023-09-22T23:33:00.402Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040084645Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3232023-09-22T23:33:00.405Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (ca1663e2-ea68-4cd9-8d69-29a1edc38cb6) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040090752Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3242023-09-22T23:33:00.408Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040097801Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3252023-09-22T23:33:00.411Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:59.040103481Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3262023-09-22T23:33:00.414Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040128967Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3272023-09-22T23:33:00.417Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040135798Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3282023-09-22T23:33:00.420Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040197779Z","hostname":"ip-10-150-1-147.us-west-2.compute.internalThe guest has finished waiting for activation
3292023-09-22T23:33:00.423Z","pid":1177}
3302023-09-22T23:33:00.425Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040213966Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3312023-09-22T23:33:00.428Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040224173Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3322023-09-22T23:33:00.431Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040235278Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3332023-09-22T23:33:00.434Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040244928Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3342023-09-22T23:33:00.437Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040254781Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3352023-09-22T23:33:00.439Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040263851Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3362023-09-22T23:33:00.442Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040305438Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3372023-09-22T23:33:00.445Z{"msg":"Generation requested: 2 >= found:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040315998Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3382023-09-22T23:33:00.448Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040322251Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3392023-09-22T23:33:00.451Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040327747Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3402023-09-22T23:33:00.454Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040333497Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3412023-09-22T23:33:00.460Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040338864Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3422023-09-22T23:33:00.464Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040344374Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3432023-09-22T23:33:00.467Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040350174Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3442023-09-22T23:33:00.470Z{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: ca1663e2-ea68-4cd9-8d69-29a1edc38cb6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040356338Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3452023-09-22T23:33:00.473Z{"msg":"00000000-0000-0000-0000-000000000000 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040361849Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3462023-09-22T23:33:00.476Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040367342Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3472023-09-22T23:33:00.478Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040372695Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3482023-09-22T23:33:00.481Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040378642Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3492023-09-22T23:33:00.484Z{"msg":"00000000-0000-0000-0000-000000000000 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040384465Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3502023-09-22T23:33:00.487Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040390712Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3512023-09-22T23:33:00.490Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040396386Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3522023-09-22T23:33:00.492Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.04040208Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3532023-09-22T23:33:00.495Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040407939Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3542023-09-22T23:33:00.498Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040413626Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3552023-09-22T23:33:00.502Z{"msg":"00000000-0000-0000-0000-000000000000 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040419261Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3562023-09-22T23:33:00.505Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040426211Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3572023-09-22T23:33:00.507Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040437443Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3582023-09-22T23:33:00.510Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.04044694Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3592023-09-22T23:33:00.513Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040456962Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3602023-09-22T23:33:00.516Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.040467459Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3612023-09-22T23:33:00.519Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.04047559Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3622023-09-22T23:33:00.523Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:59.04048158Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3632023-09-22T23:33:01.234Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:00.040503822Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3642023-09-22T23:33:01.237Z{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:00.040543551Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3652023-09-22T23:33:49.885Z{"msg":"Request to deactivate this guest","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.69344999Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3662023-09-22T23:33:49.889Z{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.693490382Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3672023-09-22T23:33:49.892Z{"msg":"[2] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.693717298Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3682023-09-22T23:33:49.895Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (ca1663e2-ea68-4cd9-8d69-29a1edc38cb6) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.693750234Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3692023-09-22T23:33:49.898Z{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.693757784Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3702023-09-22T23:33:49.900Z{"msg":"127.0.0.1:8830: proc: [2] client work task ended, Ok(Err([2] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:33:48.693916929Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"2"}
3712023-09-22T23:33:49.903Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.693963638Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3722023-09-22T23:33:49.906Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.693976346Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3732023-09-22T23:33:49.909Z{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.69402031Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3742023-09-22T23:33:49.911Z{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694030884Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3752023-09-22T23:33:49.914Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694040324Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3762023-09-22T23:33:49.917Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8830 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694049561Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"2"}
3772023-09-22T23:33:49.919Z{"msg":"[2] 127.0.0.1:8830 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694061007Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3782023-09-22T23:33:49.922Z{"msg":"00000000-0000-0000-0000-000000000000 Active Active New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694072247Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3792023-09-22T23:33:49.925Z{"msg":"[2] 127.0.0.1:8830 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694082628Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3802023-09-22T23:33:49.927Z{"msg":"[1] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694119676Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3812023-09-22T23:33:49.930Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (ca1663e2-ea68-4cd9-8d69-29a1edc38cb6) Active Active New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694130835Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3822023-09-22T23:33:49.933Z{"msg":"[1] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694141557Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3832023-09-22T23:33:49.936Z{"msg":"127.0.0.1:8820: proc: [1] client work task ended, Ok(Err([1] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:33:48.694150998Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"1"}
3842023-09-22T23:33:49.938Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694162459Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3852023-09-22T23:33:49.941Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694170955Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3862023-09-22T23:33:49.944Z{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694177247Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3872023-09-22T23:33:49.948Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694183155Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3882023-09-22T23:33:49.950Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694189109Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3892023-09-22T23:33:49.953Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8820 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.69419854Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"1"}
3902023-09-22T23:33:49.957Z{"msg":"[1] 127.0.0.1:8820 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694207523Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3912023-09-22T23:33:49.960Z{"msg":"00000000-0000-0000-0000-000000000000 Active New New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694220391Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3922023-09-22T23:33:49.963Z{"msg":"[1] 127.0.0.1:8820 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.694226196Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3932023-09-22T23:33:49.965Z{"msg":"[0] deactivate job 3057 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.695612562Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3942023-09-22T23:33:49.968Z{"msg":"[0] deactivate job 3058 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.706863623Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3952023-09-22T23:33:49.971Z{"msg":"[0] deactivate job 3059 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.718254219Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3962023-09-22T23:33:49.974Z{"msg":"[0] deactivate job 3060 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.728884726Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3972023-09-22T23:33:49.977Z{"msg":"[0] deactivate job 3061 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.740578379Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3982023-09-22T23:33:49.979Z{"msg":"[0] deactivate job 3062 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.751244255Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
3992023-09-22T23:33:50.164Z{"msg":"[0] deactivate job 3063 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972341973Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4002023-09-22T23:33:50.167Z{"msg":"[0] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972385226Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4012023-09-22T23:33:50.170Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (ca1663e2-ea68-4cd9-8d69-29a1edc38cb6) Active New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972393256Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4022023-09-22T23:33:50.173Z{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972402817Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4032023-09-22T23:33:50.175Z{"msg":"127.0.0.1:8810: proc: [0] client work task ended, Ok(Err([0] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:33:48.972533536Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"0"}
4042023-09-22T23:33:50.178Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972578678Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4052023-09-22T23:33:50.181Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972591216Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4062023-09-22T23:33:50.184Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972601996Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4072023-09-22T23:33:50.187Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972611624Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4082023-09-22T23:33:50.191Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972623346Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4092023-09-22T23:33:50.194Z{"msg":"All DS in the proper state! -> INIT","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972633538Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4102023-09-22T23:33:50.197Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8810 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972644102Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177,"looper":"0"}
4112023-09-22T23:33:50.200Z{"msg":"[0] 127.0.0.1:8810 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972655597Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4122023-09-22T23:33:50.203Z{"msg":"00000000-0000-0000-0000-000000000000 New New New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972665948Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4132023-09-22T23:33:50.205Z{"msg":"[0] 127.0.0.1:8810 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:48.972676095Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1177}
4142023-09-22T23:33:51.576Zraw options: Opt { target: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], key: None, gen: 3, cert_pem: None, key_pem: None, root_cert_pem: None, control: None, num_bytes: Some(4294967296), byte_offset: 0, iocmd_block_count: 32768, pipeline_length: 4, benchmarking_mode: None, subcommand: Write }
4152023-09-22T23:33:51.579Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.385768348Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4162023-09-22T23:33:51.582Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: false,\n opt_level: 3,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386021273Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4172023-09-22T23:33:51.585Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386071926Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4182023-09-22T23:33:51.587Z{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.38608562Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4192023-09-22T23:33:51.590Z{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: c65b7359-18c3-466e-a3d1-40130d1000ed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386097656Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4202023-09-22T23:33:51.593ZCrucible runtime is spawned
4212023-09-22T23:33:51.595Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386338093Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"0"}
4222023-09-22T23:33:51.598Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386353891Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"1"}
4232023-09-22T23:33:51.601Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible"The guest has requested activation
4242023-09-22T23:33:51.604Z,"level":30,"time":"2023-09-22T23:33:50.386509923Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"2"}
4252023-09-22T23:33:51.607Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386583954Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"task":"up_listen"}
4262023-09-22T23:33:51.610Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.3866085Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4272023-09-22T23:33:51.613Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386621237Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4282023-09-22T23:33:51.616Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386759468Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"2"}
4292023-09-22T23:33:51.619Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386812341Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"0"}
4302023-09-22T23:33:51.624Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386858084Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4312023-09-22T23:33:51.627Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.38686902Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"1"}
4322023-09-22T23:33:51.630Z{"msg":"00000000-0000-0000-0000-000000000000 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386879845Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4332023-09-22T23:33:51.633Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.386956155Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4342023-09-22T23:33:51.636Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.38709791Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4352023-09-22T23:33:51.639Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387132097Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4362023-09-22T23:33:51.641Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387139867Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4372023-09-22T23:33:51.644Z{"msg":"[2] client is_active_req TRUE, promote! session 8caf28dc-683b-4f7d-825a-f19f0da267cb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387146612Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4382023-09-22T23:33:51.647Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387153079Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4392023-09-22T23:33:51.650Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387159312Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4402023-09-22T23:33:51.652Z{"msg":"[0] client is_active_req TRUE, promote! session 8caf28dc-683b-4f7d-825a-f19f0da267cb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387169884Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4412023-09-22T23:33:51.655Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387301953Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4422023-09-22T23:33:51.658Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387417415Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4432023-09-22T23:33:51.660Z{"msg":"[1] client is_active_req TRUE, promote! session 8caf28dc-683b-4f7d-825a-f19f0da267cb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387449162Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4442023-09-22T23:33:51.663Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387501366Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4452023-09-22T23:33:51.666Z{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387527841Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4462023-09-22T23:33:51.669Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387592681Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4472023-09-22T23:33:51.671Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387677724Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4482023-09-22T23:33:51.674Z{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387707758Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4492023-09-22T23:33:51.677Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387719014Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4502023-09-22T23:33:51.680Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.38773354Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4512023-09-22T23:33:51.683Z{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387740565Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4522023-09-22T23:33:51.685Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.387821011Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4532023-09-22T23:33:51.688Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.388661623Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4542023-09-22T23:33:51.691Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.388694194Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4552023-09-22T23:33:51.694Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:50.388701241Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4562023-09-22T23:33:51.697Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.388707806Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4572023-09-22T23:33:51.699Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.388833302Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4582023-09-22T23:33:51.702Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.388877869Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4592023-09-22T23:33:51.705Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.388890041Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4602023-09-22T23:33:51.708Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.388924681Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4612023-09-22T23:33:51.710Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.388936047Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4622023-09-22T23:33:51.713Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.388969925Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4632023-09-22T23:33:51.716Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:50.388980524Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4642023-09-22T23:33:51.719Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389014401Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4652023-09-22T23:33:51.721Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389027841Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4662023-09-22T23:33:51.724Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible",The guest has finished waiting for activation
4672023-09-22T23:33:51.727Z"level":30,"time":"2023-09-22T23:33:50.389065522Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4682023-09-22T23:33:51.730Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389078075Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4692023-09-22T23:33:51.732Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:50.38912164Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4702023-09-22T23:33:51.735Z{"msg":"00000000-0000-0000-0000-000000000000 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389132743Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4712023-09-22T23:33:51.738Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389169237Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4722023-09-22T23:33:51.741Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389176804Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4732023-09-22T23:33:51.745Z{"msg":"[0]R generation[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389183725Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4742023-09-22T23:33:51.747Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389189828Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4752023-09-22T23:33:51.750Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389195608Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4762023-09-22T23:33:51.753Z{"msg":"[1]R generation[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389201516Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4772023-09-22T23:33:51.756Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389208157Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4782023-09-22T23:33:51.759Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389241393Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4792023-09-22T23:33:51.762Z{"msg":"[2]R generation[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389251946Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4802023-09-22T23:33:51.765Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389276968Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4812023-09-22T23:33:51.768Z{"msg":"Max found gen is 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389283655Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4822023-09-22T23:33:51.771Z{"msg":"Generation requested: 3 >= found:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.38928932Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4832023-09-22T23:33:51.773Z{"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.38929492Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4842023-09-22T23:33:51.776Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389300629Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4852023-09-22T23:33:51.779Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389306157Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4862023-09-22T23:33:51.782Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389311675Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4872023-09-22T23:33:51.785Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389317572Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4882023-09-22T23:33:51.788Z{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 8caf28dc-683b-4f7d-825a-f19f0da267cb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389323722Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4892023-09-22T23:33:51.791Z{"msg":"00000000-0000-0000-0000-000000000000 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389329372Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4902023-09-22T23:33:51.794Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389334878Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4912023-09-22T23:33:51.797Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389340399Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4922023-09-22T23:33:51.799Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389346596Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4932023-09-22T23:33:51.802Z{"msg":"00000000-0000-0000-0000-000000000000 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389352253Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4942023-09-22T23:33:51.805Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389358197Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4952023-09-22T23:33:51.808Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389364052Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4962023-09-22T23:33:51.811Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.38937012Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4972023-09-22T23:33:51.814Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389376404Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4982023-09-22T23:33:51.817Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389382217Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
4992023-09-22T23:33:51.820Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389388194Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5002023-09-22T23:33:51.823Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389394202Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5012023-09-22T23:33:51.825Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389400399Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5022023-09-22T23:33:51.828Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.38940634Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5032023-09-22T23:33:51.831Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.389412442Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5042023-09-22T23:33:52.582Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:51.389904594Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5052023-09-22T23:33:52.585Z{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:51.389965965Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5062023-09-22T23:38:24.383Z{"msg":"Request to deactivate this guest","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386351687Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5072023-09-22T23:38:24.386Z{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386417752Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5082023-09-22T23:38:24.389Z{"msg":"[0] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386778222Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5092023-09-22T23:38:24.392Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386837705Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5102023-09-22T23:38:24.395Z{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386851331Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5112023-09-22T23:38:24.398Z{"msg":"127.0.0.1:8810: proc: [0] client work task ended, Ok(Err([0] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:38:24.386906453Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"0"}
5122023-09-22T23:38:24.401Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386920729Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5132023-09-22T23:38:24.404Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386946449Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5142023-09-22T23:38:24.407Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386958701Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5152023-09-22T23:38:24.410Z{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386968107Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5162023-09-22T23:38:24.412Z{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386977576Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5172023-09-22T23:38:24.415Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8810 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386987477Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"0"}
5182023-09-22T23:38:24.418Z{"msg":"[0] 127.0.0.1:8810 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.386997837Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5192023-09-22T23:38:24.421Z{"msg":"[1] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387023523Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5202023-09-22T23:38:24.423Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) New Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387033963Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5212023-09-22T23:38:24.426Z{"msg":"[1] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387043587Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5222023-09-22T23:38:24.429Z{"msg":"00000000-0000-0000-0000-000000000000 New Deactivated Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387066193Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5232023-09-22T23:38:24.431Z{"msg":"[0] 127.0.0.1:8810 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387101427Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5242023-09-22T23:38:24.434Z{"msg":"127.0.0.1:8820: proc: [1] client work task ended, Ok(Err([1] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:38:24.387112266Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"1"}
5252023-09-22T23:38:24.436Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387235128Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5262023-09-22T23:38:24.439Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387246708Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5272023-09-22T23:38:24.442Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387278605Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5282023-09-22T23:38:24.445Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.38731293Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5292023-09-22T23:38:24.447Z{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.38734951Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5302023-09-22T23:38:24.450Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8820 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387359306Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"1"}
5312023-09-22T23:38:24.452Z{"msg":"[1] 127.0.0.1:8820 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387392778Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5322023-09-22T23:38:24.455Z{"msg":"00000000-0000-0000-0000-000000000000 New New Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387402904Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5332023-09-22T23:38:24.458Z{"msg":"[1] 127.0.0.1:8820 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:24.387412284Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5342023-09-22T23:38:25.385Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:25.388313397Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"0"}
5352023-09-22T23:38:25.388Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:25.388367306Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"1"}
5362023-09-22T23:38:25.390Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:25.388381438Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5372023-09-22T23:38:25.394Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:25.388393195Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5382023-09-22T23:38:25.396Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) New New Active ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:25.388528796Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5392023-09-22T23:38:25.400Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:25.388580403Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5402023-09-22T23:38:25.402Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) WaitActive New Active ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:25.388593927Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5412023-09-22T23:38:25.405Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:25.388604156Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5422023-09-22T23:38:41.002Z{"msg":"[2] deactivate job 1060 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.006900857Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5432023-09-22T23:38:41.005Z{"msg":"[2] deactivate job 1061 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.006958939Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5442023-09-22T23:38:41.019Z{"msg":"[2] deactivate job 1062 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023230336Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5452023-09-22T23:38:41.022Z{"msg":"[2] deactivate job 1063 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023288106Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5462023-09-22T23:38:41.025Z{"msg":"[2] deactivate job 1064 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023373327Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5472023-09-22T23:38:41.028Z{"msg":"[2] deactivate job 1065 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023430986Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5482023-09-22T23:38:41.031Z{"msg":"[2] deactivate job 1066 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.02344419Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5492023-09-22T23:38:41.034Z{"msg":"[2] deactivate job 1067 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023486222Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5502023-09-22T23:38:41.037Z{"msg":"[2] deactivate job 1068 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023498118Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5512023-09-22T23:38:41.040Z{"msg":"[2] deactivate job 1069 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.02354577Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5522023-09-22T23:38:41.043Z{"msg":"[2] deactivate job 1070 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023557651Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5532023-09-22T23:38:41.046Z{"msg":"[2] deactivate job 1071 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023597288Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5542023-09-22T23:38:41.049Z{"msg":"[2] deactivate job 1072 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023608625Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5552023-09-22T23:38:41.052Z{"msg":"[2] deactivate job 1073 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023645231Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5562023-09-22T23:38:41.055Z{"msg":"[2] deactivate job 1074 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.023656392Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5572023-09-22T23:38:41.058Z{"msg":"[2] deactivate job 1075 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.033265158Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5582023-09-22T23:38:41.061Z{"msg":"[2] deactivate job 1076 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.033322326Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5592023-09-22T23:38:41.064Z{"msg":"[2] deactivate job 1077 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.033399773Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5602023-09-22T23:38:41.067Z{"msg":"[2] deactivate job 1078 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.03344986Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5612023-09-22T23:38:41.070Z{"msg":"[2] deactivate job 1079 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.03346235Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5622023-09-22T23:38:41.073Z{"msg":"[2] deactivate job 1080 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.0334726Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5632023-09-22T23:38:41.076Z{"msg":"[2] deactivate job 1081 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.033519995Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5642023-09-22T23:38:41.079Z{"msg":"[2] deactivate job 1082 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.033532693Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5652023-09-22T23:38:41.082Z{"msg":"[2] deactivate job 1083 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.033569856Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5662023-09-22T23:38:41.085Z{"msg":"[2] deactivate job 1084 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.033581494Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5672023-09-22T23:38:41.088Z{"msg":"[2] deactivate job 1085 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.033618414Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5682023-09-22T23:38:41.091Z{"msg":"[2] deactivate job 1086 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.033629608Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5692023-09-22T23:38:41.094Z{"msg":"[2] deactivate job 1087 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.033664911Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5702023-09-22T23:38:41.097Z{"msg":"[2] deactivate job 1088 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045383066Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5712023-09-22T23:38:41.100Z{"msg":"[2] deactivate job 1089 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045454095Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5722023-09-22T23:38:41.103Z{"msg":"[2] deactivate job 1090 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045496057Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5732023-09-22T23:38:41.107Z{"msg":"[2] deactivate job 1091 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045508074Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5742023-09-22T23:38:41.110Z{"msg":"[2] deactivate job 1092 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045588301Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5752023-09-22T23:38:41.113Z{"msg":"[2] deactivate job 1093 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045622608Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5762023-09-22T23:38:41.116Z{"msg":"[2] deactivate job 1094 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045742544Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5772023-09-22T23:38:41.119Z{"msg":"[2] deactivate job 1095 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045816696Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5782023-09-22T23:38:41.122Z{"msg":"[2] deactivate job 1096 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045852006Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5792023-09-22T23:38:41.125Z{"msg":"[2] deactivate job 1097 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045863194Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5802023-09-22T23:38:41.128Z{"msg":"[2] deactivate job 1098 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045869698Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5812023-09-22T23:38:41.130Z{"msg":"[2] deactivate job 1099 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045929129Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5822023-09-22T23:38:41.134Z{"msg":"[2] deactivate job 1100 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045965499Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5832023-09-22T23:38:41.137Z{"msg":"[2] deactivate job 1101 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.045975987Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5842023-09-22T23:38:41.140Z{"msg":"[2] deactivate job 1102 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.046032142Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5852023-09-22T23:38:41.143Z{"msg":"[2] deactivate job 1103 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057515513Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5862023-09-22T23:38:41.146Z{"msg":"[2] deactivate job 1104 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057569441Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5872023-09-22T23:38:41.149Z{"msg":"[2] deactivate job 1105 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057595431Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5882023-09-22T23:38:41.152Z{"msg":"[2] deactivate job 1106 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057692132Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5892023-09-22T23:38:41.155Z{"msg":"[2] deactivate job 1107 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057727422Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5902023-09-22T23:38:41.158Z{"msg":"[2] deactivate job 1108 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.05773862Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5912023-09-22T23:38:41.161Z{"msg":"[2] deactivate job 1109 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057751454Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5922023-09-22T23:38:41.164Z{"msg":"[2] deactivate job 1110 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057761663Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5932023-09-22T23:38:41.167Z{"msg":"[2] deactivate job 1111 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057803688Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5942023-09-22T23:38:41.170Z{"msg":"[2] deactivate job 1112 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057813966Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5952023-09-22T23:38:41.173Z{"msg":"[2] deactivate job 1113 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057825291Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5962023-09-22T23:38:41.176Z{"msg":"[2] deactivate job 1114 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057835215Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5972023-09-22T23:38:41.179Z{"msg":"[2] deactivate job 1115 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057863719Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5982023-09-22T23:38:41.182Z{"msg":"[2] deactivate job 1116 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.057873527Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
5992023-09-22T23:38:41.185Z{"msg":"[2] deactivate job 1117 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068550216Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6002023-09-22T23:38:41.188Z{"msg":"[2] deactivate job 1118 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068600063Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6012023-09-22T23:38:41.191Z{"msg":"[2] deactivate job 1119 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068611378Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6022023-09-22T23:38:41.194Z{"msg":"[2] deactivate job 1120 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068619909Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6032023-09-22T23:38:41.197Z{"msg":"[2] deactivate job 1121 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068675107Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6042023-09-22T23:38:41.200Z{"msg":"[2] deactivate job 1122 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.06868524Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6052023-09-22T23:38:41.203Z{"msg":"[2] deactivate job 1123 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068697115Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6062023-09-22T23:38:41.205Z{"msg":"[2] deactivate job 1124 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068707086Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6072023-09-22T23:38:41.208Z{"msg":"[2] deactivate job 1125 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068735421Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6082023-09-22T23:38:41.211Z{"msg":"[2] deactivate job 1126 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068745943Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6092023-09-22T23:38:41.214Z{"msg":"[2] deactivate job 1127 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068756587Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6102023-09-22T23:38:41.218Z{"msg":"[2] deactivate job 1128 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068765967Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6112023-09-22T23:38:41.221Z{"msg":"[2] deactivate job 1129 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068795192Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6122023-09-22T23:38:41.224Z{"msg":"[2] deactivate job 1130 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.068804922Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6132023-09-22T23:38:41.227Z{"msg":"[2] deactivate job 1131 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.07950586Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6142023-09-22T23:38:41.230Z{"msg":"[2] deactivate job 1132 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079525499Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6152023-09-22T23:38:41.232Z{"msg":"[2] deactivate job 1133 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079535518Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6162023-09-22T23:38:41.235Z{"msg":"[2] deactivate job 1134 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079549523Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6172023-09-22T23:38:41.238Z{"msg":"[2] deactivate job 1135 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079559192Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6182023-09-22T23:38:41.241Z{"msg":"[2] deactivate job 1136 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079569549Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6192023-09-22T23:38:41.244Z{"msg":"[2] deactivate job 1137 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079578688Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6202023-09-22T23:38:41.247Z{"msg":"[2] deactivate job 1138 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.07958807Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6212023-09-22T23:38:41.250Z{"msg":"[2] deactivate job 1139 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079598602Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6222023-09-22T23:38:41.253Z{"msg":"[2] deactivate job 1140 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079607966Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6232023-09-22T23:38:41.256Z{"msg":"[2] deactivate job 1141 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.07961815Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6242023-09-22T23:38:41.259Z{"msg":"[2] deactivate job 1142 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079627091Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6252023-09-22T23:38:41.262Z{"msg":"[2] deactivate job 1143 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079632807Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6262023-09-22T23:38:41.264Z{"msg":"[2] deactivate job 1144 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079638552Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6272023-09-22T23:38:41.267Z{"msg":"[2] deactivate job 1145 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.079668542Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6282023-09-22T23:38:41.270Z{"msg":"[2] deactivate job 1146 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091559076Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6292023-09-22T23:38:41.273Z{"msg":"[2] deactivate job 1147 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091600676Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6302023-09-22T23:38:41.276Z{"msg":"[2] deactivate job 1148 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091607177Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6312023-09-22T23:38:41.279Z{"msg":"[2] deactivate job 1149 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091613466Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6322023-09-22T23:38:41.281Z{"msg":"[2] deactivate job 1150 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091619344Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6332023-09-22T23:38:41.284Z{"msg":"[2] deactivate job 1151 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.09162519Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6342023-09-22T23:38:41.287Z{"msg":"[2] deactivate job 1152 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091631778Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6352023-09-22T23:38:41.290Z{"msg":"[2] deactivate job 1153 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091638228Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6362023-09-22T23:38:41.293Z{"msg":"[2] deactivate job 1154 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091643752Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6372023-09-22T23:38:41.296Z{"msg":"[2] deactivate job 1155 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091649752Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6382023-09-22T23:38:41.299Z{"msg":"[2] deactivate job 1156 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091656574Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6392023-09-22T23:38:41.302Z{"msg":"[2] deactivate job 1157 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091662391Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6402023-09-22T23:38:41.305Z{"msg":"[2] deactivate job 1158 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.091668104Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6412023-09-22T23:38:41.307Z{"msg":"[2] deactivate job 1159 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101440852Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6422023-09-22T23:38:41.311Z{"msg":"[2] deactivate job 1160 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101459467Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6432023-09-22T23:38:41.313Z{"msg":"[2] deactivate job 1161 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101465445Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6442023-09-22T23:38:41.316Z{"msg":"[2] deactivate job 1162 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101471113Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6452023-09-22T23:38:41.319Z{"msg":"[2] deactivate job 1163 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101477146Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6462023-09-22T23:38:41.322Z{"msg":"[2] deactivate job 1164 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101483031Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6472023-09-22T23:38:41.325Z{"msg":"[2] deactivate job 1165 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101488729Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6482023-09-22T23:38:41.327Z{"msg":"[2] deactivate job 1166 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101494459Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6492023-09-22T23:38:41.330Z{"msg":"[2] deactivate job 1167 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101499964Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6502023-09-22T23:38:41.333Z{"msg":"[2] deactivate job 1168 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101505496Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6512023-09-22T23:38:41.336Z{"msg":"[2] deactivate job 1169 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.10151183Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6522023-09-22T23:38:41.339Z{"msg":"[2] deactivate job 1170 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101518277Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6532023-09-22T23:38:41.341Z{"msg":"[2] deactivate job 1171 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.10152394Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6542023-09-22T23:38:41.344Z{"msg":"[2] deactivate job 1172 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.101529975Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6552023-09-22T23:38:41.347Z{"msg":"[2] deactivate job 1173 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112102117Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6562023-09-22T23:38:41.350Z{"msg":"[2] deactivate job 1174 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112133676Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6572023-09-22T23:38:41.353Z{"msg":"[2] deactivate job 1175 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112140437Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6582023-09-22T23:38:41.356Z{"msg":"[2] deactivate job 1176 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112146148Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6592023-09-22T23:38:41.359Z{"msg":"[2] deactivate job 1177 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112151767Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6602023-09-22T23:38:41.361Z{"msg":"[2] deactivate job 1178 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112157402Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6612023-09-22T23:38:41.364Z{"msg":"[2] deactivate job 1179 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112163007Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6622023-09-22T23:38:41.367Z{"msg":"[2] deactivate job 1180 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112168574Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6632023-09-22T23:38:41.370Z{"msg":"[2] deactivate job 1181 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112174165Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6642023-09-22T23:38:41.373Z{"msg":"[2] deactivate job 1182 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112179895Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6652023-09-22T23:38:41.375Z{"msg":"[2] deactivate job 1183 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112185944Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6662023-09-22T23:38:41.378Z{"msg":"[2] deactivate job 1184 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112194654Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6672023-09-22T23:38:41.380Z{"msg":"[2] deactivate job 1185 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.112200319Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6682023-09-22T23:38:41.383Z{"msg":"[2] deactivate job 1186 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.121810394Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6692023-09-22T23:38:41.386Z{"msg":"[2] deactivate job 1187 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.121826861Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6702023-09-22T23:38:41.389Z{"msg":"[2] deactivate job 1188 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.121832795Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6712023-09-22T23:38:41.392Z{"msg":"[2] deactivate job 1189 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.12183841Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6722023-09-22T23:38:41.395Z{"msg":"[2] deactivate job 1190 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.121844244Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6732023-09-22T23:38:41.397Z{"msg":"[2] deactivate job 1191 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.121849796Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6742023-09-22T23:38:41.400Z{"msg":"[2] deactivate job 1192 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.121855395Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6752023-09-22T23:38:41.403Z{"msg":"[2] deactivate job 1193 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.12186075Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6762023-09-22T23:38:41.406Z{"msg":"[2] deactivate job 1194 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.121866225Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6772023-09-22T23:38:41.409Z{"msg":"[2] deactivate job 1195 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.121871627Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6782023-09-22T23:38:41.412Z{"msg":"[2] deactivate job 1196 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.121877041Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6792023-09-22T23:38:41.415Z{"msg":"[2] deactivate job 1197 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.12188241Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6802023-09-22T23:38:41.417Z{"msg":"[2] deactivate job 1198 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130553902Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6812023-09-22T23:38:41.420Z{"msg":"[2] deactivate job 1199 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130569259Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6822023-09-22T23:38:41.423Z{"msg":"[2] deactivate job 1200 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130575475Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6832023-09-22T23:38:41.426Z{"msg":"[2] deactivate job 1201 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130581451Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6842023-09-22T23:38:41.429Z{"msg":"[2] deactivate job 1202 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130587028Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6852023-09-22T23:38:41.432Z{"msg":"[2] deactivate job 1203 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130593222Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6862023-09-22T23:38:41.435Z{"msg":"[2] deactivate job 1204 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130598943Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6872023-09-22T23:38:41.438Z{"msg":"[2] deactivate job 1205 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130604652Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6882023-09-22T23:38:41.441Z{"msg":"[2] deactivate job 1206 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130611377Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6892023-09-22T23:38:41.446Z{"msg":"[2] deactivate job 1207 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130617306Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6902023-09-22T23:38:41.449Z{"msg":"[2] deactivate job 1208 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130622988Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6912023-09-22T23:38:41.452Z{"msg":"[2] deactivate job 1209 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.1306288Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6922023-09-22T23:38:41.455Z{"msg":"[2] deactivate job 1210 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130635201Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6932023-09-22T23:38:41.458Z{"msg":"[2] deactivate job 1211 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.130641117Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6942023-09-22T23:38:41.460Z{"msg":"[2] deactivate job 1212 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141086594Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6952023-09-22T23:38:41.464Z{"msg":"[2] deactivate job 1213 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141103154Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6962023-09-22T23:38:41.467Z{"msg":"[2] deactivate job 1214 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141108935Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6972023-09-22T23:38:41.470Z{"msg":"[2] deactivate job 1215 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141115036Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6982023-09-22T23:38:41.472Z{"msg":"[2] deactivate job 1216 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141121484Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
6992023-09-22T23:38:41.475Z{"msg":"[2] deactivate job 1217 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141127626Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7002023-09-22T23:38:41.498Z{"msg":"[2] deactivate job 1218 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141133459Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7012023-09-22T23:38:41.501Z{"msg":"[2] deactivate job 1219 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141139145Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7022023-09-22T23:38:41.504Z{"msg":"[2] deactivate job 1220 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141144787Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7032023-09-22T23:38:41.509Z{"msg":"[2] deactivate job 1221 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141150332Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7042023-09-22T23:38:41.513Z{"msg":"[2] deactivate job 1222 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141156232Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7052023-09-22T23:38:41.515Z{"msg":"[2] deactivate job 1223 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.141175263Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7062023-09-22T23:38:41.628Z{"msg":"[2] deactivate job 1224 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:41.632390675Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7072023-09-22T23:38:42.091Z{"msg":"[2] deactivate job 1225 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:42.095730081Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7082023-09-22T23:38:42.185Z{"msg":"[2] deactivate job 1226 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:42.189058073Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7092023-09-22T23:38:42.338Z{"msg":"[2] deactivate job 1227 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:42.341866451Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7102023-09-22T23:38:42.550Z{"msg":"[2] deactivate job 1228 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:42.550738226Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7112023-09-22T23:38:42.780Z{"msg":"[2] deactivate job 1229 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:42.783838432Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7122023-09-22T23:38:42.982Z{"msg":"[2] deactivate job 1230 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:42.986418112Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7132023-09-22T23:38:43.218Z{"msg":"[2] deactivate job 1231 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:43.222238549Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7142023-09-22T23:38:43.413Z{"msg":"[2] deactivate job 1232 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:43.417891784Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7152023-09-22T23:38:43.616Z{"msg":"[2] deactivate job 1233 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:43.620340943Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7162023-09-22T23:38:43.853Z{"msg":"[2] deactivate job 1234 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:43.857405962Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7172023-09-22T23:38:44.053Z{"msg":"[2] deactivate job 1235 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:44.05761321Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7182023-09-22T23:38:44.265Z{"msg":"[2] deactivate job 1236 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:44.269801884Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7192023-09-22T23:38:44.778Z{"msg":"[2] deactivate job 1237 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:44.782804451Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7202023-09-22T23:38:44.882Z{"msg":"[2] deactivate job 1238 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:44.886201421Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7212023-09-22T23:38:45.100Z{"msg":"[2] deactivate job 1239 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:45.104215347Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7222023-09-22T23:38:45.305Z{"msg":"[2] deactivate job 1240 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:45.309748139Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7232023-09-22T23:38:45.527Z{"msg":"[2] deactivate job 1241 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:45.531780175Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7242023-09-22T23:38:45.729Z{"msg":"[2] deactivate job 1242 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:45.733485715Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7252023-09-22T23:38:45.960Z{"msg":"[2] deactivate job 1243 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:45.964138052Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7262023-09-22T23:38:46.172Z{"msg":"[2] deactivate job 1244 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:46.176262832Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7272023-09-22T23:38:46.407Z{"msg":"[2] deactivate job 1245 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:46.411182029Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7282023-09-22T23:38:46.614Z{"msg":"[2] deactivate job 1246 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:46.618109475Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7292023-09-22T23:38:46.836Z{"msg":"[2] deactivate job 1247 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:46.840105247Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7302023-09-22T23:38:47.044Z{"msg":"[2] deactivate job 1248 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:47.048679131Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7312023-09-22T23:38:47.260Z{"msg":"[2] deactivate job 1249 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:47.264677422Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7322023-09-22T23:38:47.486Z{"msg":"[2] deactivate job 1250 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:47.4905419Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7332023-09-22T23:38:48.116Z{"msg":"[2] deactivate job 1251 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:48.120350402Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7342023-09-22T23:38:48.212Z{"msg":"[2] deactivate job 1252 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:48.216262576Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7352023-09-22T23:38:48.455Z{"msg":"[2] deactivate job 1253 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:48.459792736Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7362023-09-22T23:38:48.685Z{"msg":"[2] deactivate job 1254 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:48.689506011Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7372023-09-22T23:38:48.929Z{"msg":"[2] deactivate job 1255 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:48.933311654Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7382023-09-22T23:38:49.167Z{"msg":"[2] deactivate job 1256 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:49.17138023Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7392023-09-22T23:38:49.410Z{"msg":"[2] deactivate job 1257 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:49.414986597Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7402023-09-22T23:38:49.618Z{"msg":"[2] deactivate job 1258 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:49.622704099Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7412023-09-22T23:38:49.835Z{"msg":"[2] deactivate job 1259 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:49.839134998Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7422023-09-22T23:38:50.073Z{"msg":"[2] deactivate job 1260 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:50.077907964Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7432023-09-22T23:38:50.280Z{"msg":"[2] deactivate job 1261 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:50.284388259Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7442023-09-22T23:38:50.500Z{"msg":"[2] deactivate job 1262 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:50.504189653Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7452023-09-22T23:38:50.714Z{"msg":"[2] deactivate job 1263 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:50.718541157Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7462023-09-22T23:38:50.944Z{"msg":"[2] deactivate job 1264 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:50.949207149Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7472023-09-22T23:38:51.809Z{"msg":"[2] deactivate job 1265 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:51.81388205Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7482023-09-22T23:38:52.410Z{"msg":"[2] deactivate job 1266 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:52.414252988Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7492023-09-22T23:38:52.501Z{"msg":"[2] deactivate job 1267 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:52.506105485Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7502023-09-22T23:38:52.743Z{"msg":"[2] deactivate job 1268 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:52.747826527Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7512023-09-22T23:38:52.974Z{"msg":"[2] deactivate job 1269 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:52.978652986Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7522023-09-22T23:38:53.190Z{"msg":"[2] deactivate job 1270 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:53.194831252Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7532023-09-22T23:38:53.417Z{"msg":"[2] deactivate job 1271 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:53.421893613Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7542023-09-22T23:38:53.622Z{"msg":"[2] deactivate job 1272 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:53.626788356Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7552023-09-22T23:38:53.855Z{"msg":"[2] deactivate job 1273 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:53.859218257Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7562023-09-22T23:38:54.072Z{"msg":"[2] deactivate job 1274 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.076280775Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7572023-09-22T23:38:54.297Z{"msg":"[2] deactivate job 1275 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.3017134Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7582023-09-22T23:38:54.847Z{"msg":"[2] deactivate job 1276 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851186816Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7592023-09-22T23:38:54.849Z{"msg":"[2] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851228719Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7602023-09-22T23:38:54.853Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (8caf28dc-683b-4f7d-825a-f19f0da267cb) WaitActive WaitActive Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851236874Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7612023-09-22T23:38:54.855Z{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851243764Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7622023-09-22T23:38:54.858Z{"msg":"127.0.0.1:8830: proc: [2] client work task ended, Ok(Err([2] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:38:54.851367906Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"2"}
7632023-09-22T23:38:54.861Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851415888Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7642023-09-22T23:38:54.864Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851427851Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7652023-09-22T23:38:54.867Z{"msg":"deactivate_transition WaitActive Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.85144004Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7662023-09-22T23:38:54.870Z{"msg":"deactivate_transition WaitActive Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851450772Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7672023-09-22T23:38:54.872Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851460381Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7682023-09-22T23:38:54.875Z{"msg":"All DS in the proper state! -> INIT","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851470732Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7692023-09-22T23:38:54.878Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8830 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851481106Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180,"looper":"2"}
7702023-09-22T23:38:54.881Z{"msg":"[2] 127.0.0.1:8830 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851502649Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7712023-09-22T23:38:54.883Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851514568Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7722023-09-22T23:38:54.886Z{"msg":"[2] 127.0.0.1:8830 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:54.851524988Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1180}
7732023-09-22T23:38:56.298Zraw options: Opt { target: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], key: None, gen: 4, cert_pem: None, key_pem: None, root_cert_pem: None, control: None, num_bytes: Some(4294967296), byte_offset: 0, iocmd_block_count: 4096, pipeline_length: 8, benchmarking_mode: Some("/tmp/crudd-benchmark-num-bytes-processed.txt"), subcommand: Read }
7742023-09-22T23:38:56.301Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.304772691Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7752023-09-22T23:38:56.304Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: false,\n opt_level: 3,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.304979244Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7762023-09-22T23:38:56.307Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305006987Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7772023-09-22T23:38:56.310Z{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305013953Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7782023-09-22T23:38:56.313Z{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 125010a1-47a0-45ff-b557-38a6be92ebe2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305021136Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7792023-09-22T23:38:56.315ZCrucible runtime is spawned
7802023-09-22T23:38:56.318Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30The guest has requested activation
7812023-09-22T23:38:56.320Z,"time":"2023-09-22T23:38:56.305139271Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"1"}
7822023-09-22T23:38:56.323Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305409987Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"0"}
7832023-09-22T23:38:56.326Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.30542702Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"2"}
7842023-09-22T23:38:56.328Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305470524Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"task":"up_listen"}
7852023-09-22T23:38:56.331Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.30549261Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7862023-09-22T23:38:56.334Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305508138Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7872023-09-22T23:38:56.337Z{"msg":"00000000-0000-0000-0000-000000000000 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305517996Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7882023-09-22T23:38:56.340Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305528359Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"1"}
7892023-09-22T23:38:56.343Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305564671Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7902023-09-22T23:38:56.345Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305717452Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"2"}
7912023-09-22T23:38:56.348Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305763725Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"0"}
7922023-09-22T23:38:56.351Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305774186Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7932023-09-22T23:38:56.354Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305780274Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7942023-09-22T23:38:56.357Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305787392Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7952023-09-22T23:38:56.360Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305798214Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7962023-09-22T23:38:56.363Z{"msg":"[1] client is_active_req TRUE, promote! session 5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.305808587Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7972023-09-22T23:38:56.365Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.30591531Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7982023-09-22T23:38:56.368Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.306029706Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
7992023-09-22T23:38:56.371Z{"msg":"[0] client is_active_req TRUE, promote! session 5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.30607507Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8002023-09-22T23:38:56.374Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.3060859Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8012023-09-22T23:38:56.376Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.306093073Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8022023-09-22T23:38:56.379Z{"msg":"[2] client is_active_req TRUE, promote! session 5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.306099066Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8032023-09-22T23:38:56.382Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.306105131Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8042023-09-22T23:38:56.385Z{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.306117738Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8052023-09-22T23:38:56.387Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.30620244Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8062023-09-22T23:38:56.390Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.30624836Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8072023-09-22T23:38:56.393Z{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.306259987Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8082023-09-22T23:38:56.396Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.306267531Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8092023-09-22T23:38:56.399Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.306276905Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8102023-09-22T23:38:56.401Z{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.306283138Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8112023-09-22T23:38:56.405Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.306353967Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8122023-09-22T23:38:56.408Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.30725857Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8132023-09-22T23:38:56.410Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307299302Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8142023-09-22T23:38:56.413Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:38:56.307307274Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8152023-09-22T23:38:56.416Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307383775Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8162023-09-22T23:38:56.418Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307425922Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8172023-09-22T23:38:56.422Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307433585Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8182023-09-22T23:38:56.424Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:38:56.307439768Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8192023-09-22T23:38:56.427Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307446064Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8202023-09-22T23:38:56.430Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307453253Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8212023-09-22T23:38:56.432Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307459657Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8222023-09-22T23:38:56.435Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:38:56.307465373Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8232023-09-22T23:38:56.438Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307498424Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8242023-09-22T23:38:56.440Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307505576Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8252023-09-22T23:38:56.443Z{"msg":"[0]R flush_numbers[0..12]: [16, 17, 17, 18, 18, 19, 19, 20, 21, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307523219Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8262023-09-22T23:38:56.446Z{"msg":"[0]R generation[0..12]: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307554684Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8272023-09-22T23:38:56.449Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307566427Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8282023-09-22T23:38:56.451Z{"msg":"[1]R flush_numbers[0..12]: [16, 17, 17, 18, 18, 19, 19, 20, 21, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307577819Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8292023-09-22T23:38:56.455Z{"msg":"[1]R generation[0..12]: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307588911Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8302023-09-22T23:38:56.457Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307621668Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8312023-09-22T23:38:56.460Z{"msg":"[2]R flush_numbers[0..12]: [16, 17, 17, 18, 18, 19, 19, 20, 21, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307632711Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8322023-09-22T23:38:56.463Z{"msg":"[2]R generation[0..12]: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307645231Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8332023-09-22T23:38:56.466Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307677949Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8342023-09-22T23:38:56.468Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307688443Z","hostname":"The guest has finished waiting for activation
8352023-09-22T23:38:56.471Zip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8362023-09-22T23:38:56.474Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307703641Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8372023-09-22T23:38:56.477Z{"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307713068Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8382023-09-22T23:38:56.479Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307755895Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8392023-09-22T23:38:56.482Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307766899Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8402023-09-22T23:38:56.485Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307776353Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8412023-09-22T23:38:56.487Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307786057Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8422023-09-22T23:38:56.490Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307795648Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8432023-09-22T23:38:56.493Z{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307805371Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8442023-09-22T23:38:56.496Z{"msg":"00000000-0000-0000-0000-000000000000 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307814932Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8452023-09-22T23:38:56.499Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307824224Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8462023-09-22T23:38:56.502Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307833582Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8472023-09-22T23:38:56.505Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307841138Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8482023-09-22T23:38:56.508Z{"msg":"00000000-0000-0000-0000-000000000000 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307846754Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8492023-09-22T23:38:56.511Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307852288Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8502023-09-22T23:38:56.513Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307857827Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8512023-09-22T23:38:56.516Z{"msg":"00000000-0000-0000-0000-000000000000 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307863696Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8522023-09-22T23:38:56.519Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307869175Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8532023-09-22T23:38:56.521Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307874955Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8542023-09-22T23:38:56.524Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307881096Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8552023-09-22T23:38:56.527Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307886936Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8562023-09-22T23:38:56.530Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307892756Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8572023-09-22T23:38:56.533Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307898542Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8582023-09-22T23:38:56.535Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307904513Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8592023-09-22T23:38:56.538Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307910782Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8602023-09-22T23:38:56.541Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.307916547Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8612023-09-22T23:38:56.544Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:56.30792262Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8622023-09-22T23:38:57.303Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:57.308016994Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8632023-09-22T23:38:57.307Z{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:38:57.308076886Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8642023-09-22T23:39:46.249Z{"msg":"Request to deactivate this guest","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.255381138Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8652023-09-22T23:39:46.252Z{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.257813826Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8662023-09-22T23:39:46.255Z{"msg":"[1] deactivate job 3146 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.258000847Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8672023-09-22T23:39:46.258Z{"msg":"[2] deactivate job 3146 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.259231831Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8682023-09-22T23:39:46.260Z{"msg":"[0] deactivate job 3147 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260211427Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8692023-09-22T23:39:46.263Z{"msg":"[1] deactivate job 3147 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260649Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8702023-09-22T23:39:46.266Z{"msg":"[2] deactivate job 3147 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260675522Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8712023-09-22T23:39:46.268Z{"msg":"[0] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260682278Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8722023-09-22T23:39:46.271Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260689403Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8732023-09-22T23:39:46.274Z{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260698263Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8742023-09-22T23:39:46.277Z{"msg":"[1] deactivate, no work so YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260846901Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8752023-09-22T23:39:46.280Z{"msg":"127.0.0.1:8810: proc: [0] client work task ended, Ok(Err([0] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:39:46.260894901Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"0"}
8762023-09-22T23:39:46.283Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd) Deactivated Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260909216Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8772023-09-22T23:39:46.285Z{"msg":"[1] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260920194Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8782023-09-22T23:39:46.288Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260931345Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8792023-09-22T23:39:46.291Z{"msg":"[2] deactivate, no work so YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260942069Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8802023-09-22T23:39:46.296Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (5d95e8f2-7153-4e15-a2f2-e2f7d1e84bbd) New Deactivated Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260953373Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8812023-09-22T23:39:46.301Z{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.260964017Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8822023-09-22T23:39:46.304Z{"msg":"127.0.0.1:8830: proc: [2] client work task ended, Ok(Err([2] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:39:46.260997224Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"2"}
8832023-09-22T23:39:46.306Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261009024Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8842023-09-22T23:39:46.309Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261019624Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8852023-09-22T23:39:46.312Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261030765Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8862023-09-22T23:39:46.315Z{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261091822Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8872023-09-22T23:39:46.318Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261135623Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8882023-09-22T23:39:46.321Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8810 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261145751Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"0"}
8892023-09-22T23:39:46.323Z{"msg":"[0] 127.0.0.1:8810 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261164661Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8902023-09-22T23:39:46.326Z{"msg":"00000000-0000-0000-0000-000000000000 New Deactivated New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261200753Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8912023-09-22T23:39:46.329Z{"msg":"[0] 127.0.0.1:8810 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261211382Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8922023-09-22T23:39:46.332Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.26122033Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8932023-09-22T23:39:46.335Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261230129Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8942023-09-22T23:39:46.338Z{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261239351Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8952023-09-22T23:39:46.341Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261250469Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8962023-09-22T23:39:46.344Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8830 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261257528Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"2"}
8972023-09-22T23:39:46.347Z{"msg":"[2] 127.0.0.1:8830 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261266144Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8982023-09-22T23:39:46.350Z{"msg":"00000000-0000-0000-0000-000000000000 New Deactivated New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261276957Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
8992023-09-22T23:39:46.353Z{"msg":"[2] 127.0.0.1:8830 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261336028Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
9002023-09-22T23:39:46.356Z{"msg":"127.0.0.1:8820: proc: [1] client work task ended, Ok(Err([1] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:39:46.261346178Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"1"}
9012023-09-22T23:39:46.360Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261356958Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
9022023-09-22T23:39:46.363Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261366981Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
9032023-09-22T23:39:46.366Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261406375Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
9042023-09-22T23:39:46.369Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261418107Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
9052023-09-22T23:39:46.372Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.26142835Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
9062023-09-22T23:39:46.375Z{"msg":"All DS in the proper state! -> INIT","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.261438716Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184}
9072023-09-22T23:39:46.378Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8820 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.26144848Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"1"}
9082023-09-22T23:39:46.380Z{"msg":"127.0.0.1:8820 Message to ds_status_tx failed: SendError { .. }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:39:46.261456334Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1184,"looper":"1"}
9092023-09-22T23:39:46.383Zraw options: Opt { target: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], key: None, gen: 5, cert_pem: None, key_pem: None, root_cert_pem: None, control: None, num_bytes: Some(4294967296), byte_offset: 0, iocmd_block_count: 4096, pipeline_length: 8, benchmarking_mode: Some("/tmp/crudd-benchmark-num-bytes-processed.txt"), subcommand: Write }
9102023-09-22T23:39:46.386Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.316600548Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9112023-09-22T23:39:46.389Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: false,\n opt_level: 3,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.316989603Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9122023-09-22T23:39:46.392Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317030562Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9132023-09-22T23:39:46.395Z{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317037532Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9142023-09-22T23:39:46.398Z{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 4c32881e-d981-48e9-8e06-706d001d0617","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317044627Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9152023-09-22T23:39:46.401ZCrucible runtime is spawned
9162023-09-22T23:39:46.404Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317136157Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"0"}
9172023-09-22T23:39:46.407Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.31720282Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"1"}
9182023-09-22T23:39:46.410ZThe guest has requested activation
9192023-09-22T23:39:46.413Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317290902Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"2"}
9202023-09-22T23:39:46.416Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317494849Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"task":"up_listen"}
9212023-09-22T23:39:46.419Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.31758525Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"1"}
9222023-09-22T23:39:46.422Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317599074Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9232023-09-22T23:39:46.425Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317634447Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"2"}
9242023-09-22T23:39:46.427Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317644084Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9252023-09-22T23:39:46.430Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317666878Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"0"}
9262023-09-22T23:39:46.433Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317673538Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9272023-09-22T23:39:46.436Z{"msg":"00000000-0000-0000-0000-000000000000 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317679425Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9282023-09-22T23:39:46.439Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317752085Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9292023-09-22T23:39:46.442Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317773402Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9302023-09-22T23:39:46.446Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (8bf20137-c7a5-4048-af6e-67339371d62c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317933537Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9312023-09-22T23:39:46.449Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317976329Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9322023-09-22T23:39:46.452Z{"msg":"[0] client is_active_req TRUE, promote! session 8bf20137-c7a5-4048-af6e-67339371d62c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317988922Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9332023-09-22T23:39:46.455Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (8bf20137-c7a5-4048-af6e-67339371d62c) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.317999857Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9342023-09-22T23:39:46.458Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.31801082Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9352023-09-22T23:39:46.461Z{"msg":"[1] client is_active_req TRUE, promote! session 8bf20137-c7a5-4048-af6e-67339371d62c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318020656Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9362023-09-22T23:39:46.464Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (8bf20137-c7a5-4048-af6e-67339371d62c) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318031283Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9372023-09-22T23:39:46.467Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318041329Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9382023-09-22T23:39:46.469Z{"msg":"[2] client is_active_req TRUE, promote! session 8bf20137-c7a5-4048-af6e-67339371d62c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318051295Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9392023-09-22T23:39:46.472Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318394599Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9402023-09-22T23:39:46.475Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318410781Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9412023-09-22T23:39:46.478Z{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318421235Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9422023-09-22T23:39:46.481Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318478458Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9432023-09-22T23:39:46.484Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318490157Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9442023-09-22T23:39:46.486Z{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318500359Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9452023-09-22T23:39:46.489Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318510294Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9462023-09-22T23:39:46.492Z{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 262144, shift: 9 }, extent_count: 32, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318521365Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9472023-09-22T23:39:46.494Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.318580374Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9482023-09-22T23:39:46.497Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (8bf20137-c7a5-4048-af6e-67339371d62c) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.320990231Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9492023-09-22T23:39:46.500Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321024147Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9502023-09-22T23:39:46.503Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:39:46.321034516Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9512023-09-22T23:39:46.506Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321045489Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9522023-09-22T23:39:46.509Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321145371Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9532023-09-22T23:39:46.512Z{"msg":"00000000-0000-0000-0000-000000000000 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321159739Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9542023-09-22T23:39:46.514Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321169978Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9552023-09-22T23:39:46.517Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321179647Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9562023-09-22T23:39:46.520Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (8bf20137-c7a5-4048-af6e-67339371d62c) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321190091Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9572023-09-22T23:39:46.523Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321199838Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9582023-09-22T23:39:46.525Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:39:46.321209362Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9592023-09-22T23:39:46.528Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321215556Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9602023-09-22T23:39:46.531Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321221082Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9612023-09-22T23:39:46.534Z{"msg":"00000000-0000-0000-0000-000000000000 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321227102Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9622023-09-22T23:39:46.537Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321233399Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9632023-09-22T23:39:46.540Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321239169Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9642023-09-22T23:39:46.543Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (8bf20137-c7a5-4048-af6e-67339371d62c) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321375046Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9652023-09-22T23:39:46.546Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321388772Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9662023-09-22T23:39:46.549Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:39:46.321398529Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9672023-09-22T23:39:46.552Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321409017Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9682023-09-22T23:39:46.555Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321418774Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9692023-09-22T23:39:46.558Z{"msg":"00000000-0000-0000-0000-000000000000 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321428474Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9702023-09-22T23:39:46.561Z{"msg":"[0]R flush_numbers[0..12]: [16, 17, 17, 18, 18, 19, 19, 20, 21, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321438653Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9712023-09-22T23:39:46.563Z{"msg":"[0]R generation[0..12]: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321448568Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9722023-09-22T23:39:46.566Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321458534Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9732023-09-22T23:39:46.569Z{"msg":"[1]R flush_numbers[0..12]: [16, 17, 17, 18, 18, 19, 19, 20, 21, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321468116Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9742023-09-22T23:39:46.572Z{"msg":"[1]R generation[0..12]: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321477961Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9752023-09-22T23:39:46.575Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321488492Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9762023-09-22T23:39:46.578Z{"msg":"[2]R flush_numbers[0..12]: [16, 17, 17, 18, 18, 19, 19, 20, 21, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321498492Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9772023-09-22T23:39:46.581Z{"msg":"[2]R generation[0..12]: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321508497Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9782023-09-22T23:39:46.584Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321518406Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9792023-09-22T23:39:46.587Z{"msg":"Max found gen is 4","v":0,"name":"The guest has finished waiting for activation
9802023-09-22T23:39:46.590Zcrucible","level":30,"time":"2023-09-22T23:39:46.32153149Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9812023-09-22T23:39:46.592Z{"msg":"Generation requested: 5 >= found:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321542699Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9822023-09-22T23:39:46.595Z{"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321553296Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9832023-09-22T23:39:46.598Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.3217157Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9842023-09-22T23:39:46.601Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321727103Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9852023-09-22T23:39:46.603Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321737142Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9862023-09-22T23:39:46.606Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321747251Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9872023-09-22T23:39:46.609Z{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 8bf20137-c7a5-4048-af6e-67339371d62c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321757411Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9882023-09-22T23:39:46.612Z{"msg":"00000000-0000-0000-0000-000000000000 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.32176738Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9892023-09-22T23:39:46.615Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321777303Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9902023-09-22T23:39:46.619Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321786821Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9912023-09-22T23:39:46.622Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.32179629Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9922023-09-22T23:39:46.625Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.32180638Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9932023-09-22T23:39:46.628Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321816353Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9942023-09-22T23:39:46.631Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321826672Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9952023-09-22T23:39:46.634Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321837731Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9962023-09-22T23:39:46.637Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321847486Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9972023-09-22T23:39:46.640Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321856085Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9982023-09-22T23:39:46.643Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321862122Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
9992023-09-22T23:39:46.646Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:46.321867862Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10002023-09-22T23:39:47.317Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:47.322675233Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10012023-09-22T23:39:47.320Z{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:39:47.323524708Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10022023-09-22T23:41:11.897Z{"msg":"Request to deactivate this guest","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.895406447Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10032023-09-22T23:41:11.900Z{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.895457611Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10042023-09-22T23:41:11.902Z{"msg":"[2] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.895779508Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10052023-09-22T23:41:11.905Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 (8bf20137-c7a5-4048-af6e-67339371d62c) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.89581511Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10062023-09-22T23:41:11.908Z{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.895823226Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10072023-09-22T23:41:11.911Z{"msg":"[1] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.895916708Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10082023-09-22T23:41:11.914Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 (8bf20137-c7a5-4048-af6e-67339371d62c) Active Active Deactivated ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.895964215Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10092023-09-22T23:41:11.916Z{"msg":"[1] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.895975258Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10102023-09-22T23:41:11.919Z{"msg":"127.0.0.1:8830: proc: [2] client work task ended, Ok(Err([2] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:41:11.895986431Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"2"}
10112023-09-22T23:41:11.922Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896029197Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10122023-09-22T23:41:11.925Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896040357Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10132023-09-22T23:41:11.928Z{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896050014Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10142023-09-22T23:41:11.930Z{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896061439Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10152023-09-22T23:41:11.933Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.89606983Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10162023-09-22T23:41:11.936Z{"msg":"[2] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8830 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896076466Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"2"}
10172023-09-22T23:41:11.939Z{"msg":"[2] 127.0.0.1:8830 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896083108Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10182023-09-22T23:41:11.942Z{"msg":"127.0.0.1:8820: proc: [1] client work task ended, Ok(Err([1] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:41:11.89608899Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"1"}
10192023-09-22T23:41:11.945Z{"msg":"00000000-0000-0000-0000-000000000000 Active Deactivated New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896095471Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10202023-09-22T23:41:11.947Z{"msg":"[2] 127.0.0.1:8830 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896101791Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10212023-09-22T23:41:11.950Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896107597Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10222023-09-22T23:41:11.953Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896115827Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10232023-09-22T23:41:11.956Z{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896121563Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10242023-09-22T23:41:11.959Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896127114Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10252023-09-22T23:41:11.961Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896132714Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10262023-09-22T23:41:11.964Z{"msg":"[1] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8820 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896139077Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"1"}
10272023-09-22T23:41:11.967Z{"msg":"[1] 127.0.0.1:8820 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896146078Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10282023-09-22T23:41:11.969Z{"msg":"00000000-0000-0000-0000-000000000000 Active New New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896152045Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10292023-09-22T23:41:11.972Z{"msg":"[1] 127.0.0.1:8820 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.896178919Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10302023-09-22T23:41:11.975Z{"msg":"[0] deactivate job 3047 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.920535697Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10312023-09-22T23:41:11.978Z{"msg":"[0] deactivate job 3048 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.951147056Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10322023-09-22T23:41:11.981Z{"msg":"[0] deactivate job 3049 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.97324445Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10332023-09-22T23:41:11.995Z{"msg":"[0] deactivate job 3050 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:11.994755207Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10342023-09-22T23:41:12.046Z{"msg":"[0] deactivate job 3051 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.044605444Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10352023-09-22T23:41:12.069Z{"msg":"[0] deactivate job 3052 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.068305735Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10362023-09-22T23:41:12.091Z{"msg":"[0] deactivate job 3053 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.089920546Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10372023-09-22T23:41:12.121Z{"msg":"[0] deactivate job 3054 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.120220486Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10382023-09-22T23:41:12.142Z{"msg":"[0] deactivate job 3055 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.141016928Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10392023-09-22T23:41:12.164Z{"msg":"[0] deactivate job 3056 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.162628705Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10402023-09-22T23:41:12.193Z{"msg":"[0] deactivate job 3057 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.192350723Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10412023-09-22T23:41:12.214Z{"msg":"[0] deactivate job 3058 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.21361768Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10422023-09-22T23:41:12.235Z{"msg":"[0] deactivate job 3059 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.23405831Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10432023-09-22T23:41:12.265Z{"msg":"[0] deactivate job 3060 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.263980682Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10442023-09-22T23:41:12.283Z{"msg":"[0] deactivate job 3061 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.282591542Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10452023-09-22T23:41:12.307Z{"msg":"[0] deactivate job 3062 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.30583762Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10462023-09-22T23:41:12.335Z{"msg":"[0] deactivate job 3063 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.334409379Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10472023-09-22T23:41:12.354Z{"msg":"[0] deactivate job 3064 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.352744443Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10482023-09-22T23:41:12.378Z{"msg":"[0] deactivate job 3065 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.376896116Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10492023-09-22T23:41:12.406Z{"msg":"[0] deactivate job 3066 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.405517938Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10502023-09-22T23:41:12.442Z{"msg":"[0] deactivate job 3067 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.441353721Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10512023-09-22T23:41:12.886Z{"msg":"[0] deactivate job 3068 not InProgress flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884198827Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10522023-09-22T23:41:12.889Z{"msg":"[0] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884239505Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10532023-09-22T23:41:12.892Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 (8bf20137-c7a5-4048-af6e-67339371d62c) Active New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884247164Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10542023-09-22T23:41:12.894Z{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884254203Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10552023-09-22T23:41:12.897Z{"msg":"127.0.0.1:8810: proc: [0] client work task ended, Ok(Err([0] exits after deactivation)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:41:12.884450547Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"0"}
10562023-09-22T23:41:12.900Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884470813Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10572023-09-22T23:41:12.903Z{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884529137Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10582023-09-22T23:41:12.906Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884541823Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10592023-09-22T23:41:12.909Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884551285Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10602023-09-22T23:41:12.912Z{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884560084Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10612023-09-22T23:41:12.915Z{"msg":"All DS in the proper state! -> INIT","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884572205Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10622023-09-22T23:41:12.918Z{"msg":"[0] 00000000-0000-0000-0000-000000000000 connection to 127.0.0.1:8810 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884583584Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190,"looper":"0"}
10632023-09-22T23:41:12.921Z{"msg":"[0] 127.0.0.1:8810 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884595133Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10642023-09-22T23:41:12.924Z{"msg":"00000000-0000-0000-0000-000000000000 New New New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884606462Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10652023-09-22T23:41:12.926Z{"msg":"[0] 127.0.0.1:8810 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:41:12.884617051Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10662023-09-22T23:41:12.929Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884697623Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10672023-09-22T23:41:12.932Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884734688Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10682023-09-22T23:41:12.935Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884745954Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10692023-09-22T23:41:12.937Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884779973Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10702023-09-22T23:41:12.940Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.88478991Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10712023-09-22T23:41:12.942Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884801825Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10722023-09-22T23:41:12.945Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884812167Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10732023-09-22T23:41:12.948Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.88482405Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10742023-09-22T23:41:12.951Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884834007Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10752023-09-22T23:41:12.953Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884844034Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10762023-09-22T23:41:12.956Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884853995Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10772023-09-22T23:41:12.959Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.88486359Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10782023-09-22T23:41:12.961Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884873869Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10792023-09-22T23:41:12.964Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884883981Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10802023-09-22T23:41:12.967Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884917626Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10812023-09-22T23:41:12.970Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884927429Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10822023-09-22T23:41:12.972Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884933785Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10832023-09-22T23:41:12.975Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884939499Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10842023-09-22T23:41:12.978Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884945105Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10852023-09-22T23:41:12.980Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884950554Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10862023-09-22T23:41:12.983Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.8849597Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10872023-09-22T23:41:12.986Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884971543Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10882023-09-22T23:41:12.988Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.884981838Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10892023-09-22T23:41:12.991Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885014294Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10902023-09-22T23:41:12.994Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885024231Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10912023-09-22T23:41:12.997Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885034111Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10922023-09-22T23:41:13.000Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885066601Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10932023-09-22T23:41:13.003Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885077379Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10942023-09-22T23:41:13.006Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885115826Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10952023-09-22T23:41:13.008Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.88512663Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10962023-09-22T23:41:13.011Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.8851381Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10972023-09-22T23:41:13.015Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885147854Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10982023-09-22T23:41:13.017Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.88518239Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
10992023-09-22T23:41:13.020Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885192458Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11002023-09-22T23:41:13.023Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885201219Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11012023-09-22T23:41:13.025Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.88520676Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11022023-09-22T23:41:13.028Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885212507Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11032023-09-22T23:41:13.031Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885217989Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11042023-09-22T23:41:13.033Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885223462Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11052023-09-22T23:41:13.036Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885230267Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11062023-09-22T23:41:13.039Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885250406Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11072023-09-22T23:41:13.042Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885273507Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11082023-09-22T23:41:13.045Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885279263Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11092023-09-22T23:41:13.048Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885284921Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11102023-09-22T23:41:13.051Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885305888Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11112023-09-22T23:41:13.053Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885311567Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11122023-09-22T23:41:13.056Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.88531709Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11132023-09-22T23:41:13.059Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885322611Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11142023-09-22T23:41:13.062Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885328703Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11152023-09-22T23:41:13.064Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.88533448Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11162023-09-22T23:41:13.067Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885340593Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11172023-09-22T23:41:13.070Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885346156Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11182023-09-22T23:41:13.073Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885351772Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11192023-09-22T23:41:13.076Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885357184Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11202023-09-22T23:41:13.078Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885362683Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11212023-09-22T23:41:13.081Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885368179Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11222023-09-22T23:41:13.084Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885373648Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11232023-09-22T23:41:13.086Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885379201Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11242023-09-22T23:41:13.089Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885384595Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11252023-09-22T23:41:13.092Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885390113Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11262023-09-22T23:41:13.095Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885395767Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11272023-09-22T23:41:13.097Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885401325Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11282023-09-22T23:41:13.100Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885407046Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11292023-09-22T23:41:13.103Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885412803Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11302023-09-22T23:41:13.106Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.88541881Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11312023-09-22T23:41:13.108Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.88542447Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11322023-09-22T23:41:13.111Z{"msg":"up_listen reports status_rx -> None ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:41:12.885430293Z","hostname":"ip-10-150-1-147.us-west-2.compute.internal","pid":1190}
11332023-09-22T23:41:14.358ZShutdown 1162
11342023-09-22T23:41:14.449ZSep 22 23:41:14.447 INFO accepted connection, remote_addr: 127.0.0.1:61367, local_addr: 127.0.0.1:9998
11352023-09-22T23:41:14.452ZMain task has work to do, go find it
11362023-09-22T23:41:14.455Zgot dsc Shutdown
11372023-09-22T23:41:14.459ZShutdown
11382023-09-22T23:41:14.462ZSep 22 23:41:14.450 INFO request completedShut it down
11392023-09-22T23:41:14.464Z, latency_us: 1148, response_code: 204, uri: /shutdown, method: POST, req_id: fe2d0ae3-daae-4368-a3b5-e0239aad1b7a, remote_addr: 127.0.0.1:[8810] Disable keep_running
11402023-09-22T23:41:14.467Z61367[8810] Got stop action so:false kr:false
11412023-09-22T23:41:14.469Z, local_addr: 127.0.0.1:9998
11422023-09-22T23:41:14.472Z[8820] Disable keep_running
11432023-09-22T23:41:14.475Z[8820] Got stop action so:false kr:false
11442023-09-22T23:41:14.479Z[8830] Disable keep_running
11452023-09-22T23:41:14.481Z[8830] Got stop action so:false kr:false
11462023-09-22T23:41:14.484ZWait on pid 1162
11472023-09-22T23:41:15.454ZWaiting for all downstairs to exit
11482023-09-22T23:41:16.455ZWaiting for all downstairs to exit
11492023-09-22T23:41:16.875Z[8810] Exited with: ExitStatus(unix_wait_status(9))
11502023-09-22T23:41:16.879Z[8810][0] reports Exit
11512023-09-22T23:41:16.980Z[8820] Exited with: ExitStatus(unix_wait_status(9))
11522023-09-22T23:41:16.983Z[8820][1] reports Exit
11532023-09-22T23:41:16.986Z[8830] Exited with: ExitStatus(unix_wait_status(9))
11542023-09-22T23:41:16.989Z[8830][2] reports Exit
11552023-09-22T23:41:17.461Z1162 dead
11562023-09-22T23:41:17.464Z{ "request_size": 4096, "pipeline_length": 8, "read_speed_uninit": 95443717, "write_speed_uninit": 84215045, "read_speed_init": 85899345, "write_speed_init": 48806446 }
11572023-09-22T23:41:17.467Z
11582023-09-22T23:41:17.469Zreal 9:05.644931443
11592023-09-22T23:41:17.472Zuser 25:13.675814070
11602023-09-22T23:41:17.475Zsys 3:58.895871686
11612023-09-22T23:41:17.478Ztrap 2.102857710
11622023-09-22T23:41:17.481Ztflt 0.156642685
11632023-09-22T23:41:17.484Zdflt 0.481649463
11642023-09-22T23:41:17.486Zkflt 0.128859046
11652023-09-22T23:41:17.489Zlock 7:27:26.842987524
11662023-09-22T23:41:17.492Zslp 1:20:14.351929866
11672023-09-22T23:41:17.495Zlat 1:00.516790106
11682023-09-22T23:41:17.497Zstop 0.057731277
11692023-09-22T23:41:17.500Zprocess exited: duration 548545 ms, exit code 0
 
11702023-09-22T23:41:17.513Zfound 1 output files
11712023-09-22T23:41:17.516Zuploading: /tmp/crudd-speed-battery-results.json (169 bytes)
11722023-09-22T23:41:18.523Zuploaded: /tmp/crudd-speed-battery-results.json